From 8be14328a51c9c21ade8b954539b1dc0c556c67d Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Sun, 19 Jul 2026 16:40:39 +0800 Subject: [PATCH 1/5] test: add Rust and JS interop fuzzing --- .changeset/fuzzy-loros-agree.md | 6 + .github/workflows/rust.yml | 2 + AGENTS.md | 2 + Cargo.lock | 1 + context/loro-js-interop-fuzz.md | 34 + crates/fuzz/Cargo.toml | 1 + crates/fuzz/src/bin/loro_js_interop_driver.rs | 600 +++++++++++++ loro-js/README.md | 8 + loro-js/fuzz/.gitignore | 1 + loro-js/fuzz/README.md | 77 ++ loro-js/fuzz/corpus/core.json | 33 + loro-js/fuzz/corpus/out-of-order.json | 20 + .../fuzz/corpus/strict/richtext-anchor.json | 22 + loro-js/fuzz/run-interop-fuzz.mjs | 182 ++++ loro-js/fuzz/runner.mjs | 796 ++++++++++++++++++ loro-js/fuzz/scenario.mjs | 157 ++++ loro-js/package.json | 5 + loro-js/src/runtime/document.ts | 34 +- loro-js/tests/runtime.test.ts | 21 + package.json | 1 + pnpm-lock.yaml | 16 + 21 files changed, 1991 insertions(+), 28 deletions(-) create mode 100644 .changeset/fuzzy-loros-agree.md create mode 100644 context/loro-js-interop-fuzz.md create mode 100644 crates/fuzz/src/bin/loro_js_interop_driver.rs create mode 100644 loro-js/fuzz/.gitignore create mode 100644 loro-js/fuzz/README.md create mode 100644 loro-js/fuzz/corpus/core.json create mode 100644 loro-js/fuzz/corpus/out-of-order.json create mode 100644 loro-js/fuzz/corpus/strict/richtext-anchor.json create mode 100644 loro-js/fuzz/run-interop-fuzz.mjs create mode 100644 loro-js/fuzz/runner.mjs create mode 100644 loro-js/fuzz/scenario.mjs diff --git a/.changeset/fuzzy-loros-agree.md b/.changeset/fuzzy-loros-agree.md new file mode 100644 index 000000000..e679603f8 --- /dev/null +++ b/.changeset/fuzzy-loros-agree.md @@ -0,0 +1,6 @@ +--- +"loro-js": patch +--- + +Keep semantic no-op detection type-safe so empty arrays, objects, maps, binary +values, and containers cannot be mistaken for one another. diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 528f06694..520aef3a8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -49,3 +49,5 @@ jobs: # run: cargo clippy --all-features -- -Dwarnings - name: Run rust tests run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} deno task test-all + - name: Run Rust and TypeScript interop fuzz smoke + run: pnpm test-loro-js-interop diff --git a/AGENTS.md b/AGENTS.md index 1a9eeb31a..de186026d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,8 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [skills/loro/SKILL.md](skills/loro/SKILL.md). - Pure TypeScript runtime indexes, complexity contracts, benchmarks, and remaining gaps: [context/loro-js-performance.md](context/loro-js-performance.md). +- Rust/WASM, native Rust, and pure TypeScript differential fuzzing: + [context/loro-js-interop-fuzz.md](context/loro-js-interop-fuzz.md). - Context backlog: [context/CONTEXT-GAPS.md](context/CONTEXT-GAPS.md). ## Commands diff --git a/Cargo.lock b/Cargo.lock index 3a10d3985..1d6930342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -945,6 +945,7 @@ dependencies = [ "rand 0.8.5", "rayon", "rustc-hash", + "serde", "serde_json", "tabled 0.10.0", "tracing", diff --git a/context/loro-js-interop-fuzz.md b/context/loro-js-interop-fuzz.md new file mode 100644 index 000000000..a1205feb5 --- /dev/null +++ b/context/loro-js-interop-fuzz.md @@ -0,0 +1,34 @@ +# Rust / TypeScript interoperability fuzzing + +Verified against code 2026-07-19. + +The executable harness lives under `loro-js/fuzz/`. Its shared scenario schema +is defined in `scenario.mjs`, and `runner.mjs` is the semantic oracle for both +the Rust/WASM package and the pure TypeScript runtime. + +For every command, `runDifferentialScenario` crosses transport blobs between +the two implementations and compares canonical state, history metadata, and +events. `runMalformedImportChecks` mutates Rust- and TypeScript-produced current +update/snapshot blobs and requires matching acceptance or rejection; rejected +imports must not partially change the target. Raw encoded bytes are not compared +because current Loro encodings are wire-compatible but non-canonical. + +`crates/fuzz/src/bin/loro_js_interop_driver.rs` consumes the same JSON schema. +When `LORO_INTEROP_NATIVE_DRIVER` is set, the Node runner checks JS-produced +blobs in native Rust and native-Rust-produced blobs in JS for every permanent +corpus scenario. + +The stable generator avoids semantic no-op and invalid operations whose oplog +policy differs independently of visible CRDT behavior. It also omits rich-text +mark/unmark operations and does not compare events when a snapshot is imported +into a non-empty document. The strict corpus records these known gaps. In +particular, `fuzz/corpus/strict/richtext-anchor.json` demonstrates the remaining +anchor/entity-position mismatch after a marked range receives a later insert. + +Run the complete green smoke from the repository root with: + +```sh +pnpm test-loro-js-interop +``` + +See `loro-js/fuzz/README.md` for profiles, replay controls, and corpus promotion. diff --git a/crates/fuzz/Cargo.toml b/crates/fuzz/Cargo.toml index 3c128bb72..8502c3eaa 100644 --- a/crates/fuzz/Cargo.toml +++ b/crates/fuzz/Cargo.toml @@ -32,6 +32,7 @@ rayon = "1.10.0" bytes = "1" ensure-cov = { workspace = true } pretty_assertions = "1.4.0" +serde = { workspace = true, features = ["derive"] } loro-alpha-4 = { git = "https://github.com/loro-dev/loro.git", tag = "loro-crdt@1.0.0-alpha.4", package = "loro" } [dev-dependencies] diff --git a/crates/fuzz/src/bin/loro_js_interop_driver.rs b/crates/fuzz/src/bin/loro_js_interop_driver.rs new file mode 100644 index 000000000..1db740880 --- /dev/null +++ b/crates/fuzz/src/bin/loro_js_interop_driver.rs @@ -0,0 +1,600 @@ +use std::{error::Error, io}; + +use loro::{ + ExportMode, Frontiers, LoroDoc, LoroValue, ToJson, TreeID, TreeParentId, VersionVector, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +const SLOT_COUNT: usize = 8; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DriverInput { + scenario: Scenario, + #[serde(default)] + external_blobs: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Scenario { + schema_version: u8, + peer_count: usize, + commands: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde( + tag = "kind", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +enum Command { + MapSet { + peer: i64, + key: String, + value: Value, + }, + MapDelete { + peer: i64, + key: String, + }, + ListInsert { + peer: i64, + index: i64, + value: Value, + }, + ListDelete { + peer: i64, + index: i64, + length: i64, + }, + TextInsert { + peer: i64, + index: i64, + text: String, + }, + TextDelete { + peer: i64, + index: i64, + length: i64, + }, + TextMark { + peer: i64, + index: i64, + length: i64, + key: String, + value: Value, + }, + TextUnmark { + peer: i64, + index: i64, + length: i64, + key: String, + }, + MovableInsert { + peer: i64, + index: i64, + value: Value, + }, + MovableDelete { + peer: i64, + index: i64, + length: i64, + }, + MovableSet { + peer: i64, + index: i64, + value: Value, + }, + MovableMove { + peer: i64, + from: i64, + to: i64, + }, + CounterIncrement { + peer: i64, + delta: i64, + }, + TreeCreate { + peer: i64, + parent: Option, + value: Value, + }, + TreeMetaSet { + peer: i64, + node: i64, + key: String, + value: Value, + }, + TreeMove { + peer: i64, + node: i64, + parent: Option, + }, + TreeDelete { + peer: i64, + node: i64, + }, + Commit { + peer: i64, + message: i64, + }, + Enqueue { + source: i64, + target: i64, + slot: i64, + mode: TransportMode, + }, + Deliver { + slot: i64, + copies: i64, + }, + Save { + peer: i64, + checkpoint: i64, + }, + Checkout { + peer: i64, + checkpoint: i64, + }, + Attach { + peer: i64, + }, + Roundtrip { + peer: i64, + mode: TransportMode, + }, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum TransportMode { + Update, + Snapshot, +} + +#[derive(Debug, Clone)] +struct Slot { + blob: Vec, + target: usize, +} + +struct World { + docs: Vec, + slots: Vec>, + checkpoints: Vec>>, + external_blobs: Option>>, + transport_blobs: Vec>, + enqueue_index: usize, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DriverOutput { + observations: Vec, + transport_blobs: Vec>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Observation { + json: Value, + deep_with_id: Value, + version: Vec<(String, i32)>, + oplog_version: Vec<(String, i32)>, + frontiers: Vec, + oplog_frontiers: Vec, + detached: bool, + shallow: bool, + op_count: usize, + change_count: usize, +} + +fn main() -> Result<(), Box> { + let input: DriverInput = serde_json::from_reader(io::stdin().lock())?; + if input.scenario.schema_version != 1 { + return Err(format!( + "unsupported scenario schema {}", + input.scenario.schema_version + ) + .into()); + } + if input.scenario.peer_count < 2 { + return Err("scenario peerCount must be at least two".into()); + } + + let mut world = World::new(input.scenario.peer_count, input.external_blobs); + for command in &input.scenario.commands { + world.execute(command); + } + let output = DriverOutput { + observations: world.docs.iter().map(observe_doc).collect(), + transport_blobs: world.transport_blobs, + }; + serde_json::to_writer(io::stdout().lock(), &output)?; + Ok(()) +} + +impl World { + fn new(peer_count: usize, external_blobs: Option>>) -> Self { + let docs = (0..peer_count) + .map(|peer| { + let doc = LoroDoc::new(); + doc.set_peer_id(peer as u64 + 1).unwrap(); + doc.set_record_timestamp(false); + doc.set_change_merge_interval(0); + doc.get_map("map"); + doc.get_list("list"); + doc.get_text("text"); + doc.get_movable_list("movable"); + doc.get_counter("counter"); + doc.get_tree("tree").enable_fractional_index(0); + doc + }) + .collect(); + Self { + docs, + slots: vec![None; SLOT_COUNT], + checkpoints: vec![vec![None; SLOT_COUNT]; peer_count], + external_blobs, + transport_blobs: Vec::new(), + enqueue_index: 0, + } + } + + fn execute(&mut self, command: &Command) { + match command { + Command::MapSet { peer, key, value } => { + let _ = self + .doc(*peer) + .get_map("map") + .insert(key, loro_value(value)); + } + Command::MapDelete { peer, key } => { + let map = self.doc(*peer).get_map("map"); + let mut keys = map.keys().map(|key| key.to_string()).collect::>(); + if !keys.is_empty() { + keys.sort(); + let selected = if keys.iter().any(|candidate| candidate == key) { + key + } else { + &keys[0] + }; + let _ = map.delete(selected); + } + } + Command::ListInsert { peer, index, value } => { + let list = self.doc(*peer).get_list("list"); + let _ = list.insert(modulo(*index, list.len() + 1), loro_value(value)); + } + Command::ListDelete { + peer, + index, + length, + } => { + let list = self.doc(*peer).get_list("list"); + if !list.is_empty() { + let index = modulo(*index, list.len()); + let length = 1 + modulo(*length, list.len() - index); + let _ = list.delete(index, length); + } + } + Command::TextInsert { peer, index, text } => { + let text_container = self.doc(*peer).get_text("text"); + let boundaries = utf16_boundaries(&text_container.to_string()); + let _ = + text_container.insert_utf16(boundaries[modulo(*index, boundaries.len())], text); + } + Command::TextDelete { + peer, + index, + length, + } => { + let text = self.doc(*peer).get_text("text"); + let (start, end) = selected_text_range(&text.to_string(), *index, *length); + if start != end { + let _ = text.delete_utf16(start, end - start); + } + } + Command::TextMark { + peer, + index, + length, + key, + value, + } => { + let text = self.doc(*peer).get_text("text"); + let (start, end) = selected_text_range(&text.to_string(), *index, *length); + if start != end { + let _ = text.mark_utf16(start..end, key, loro_value(value)); + } + } + Command::TextUnmark { + peer, + index, + length, + key, + } => { + let text = self.doc(*peer).get_text("text"); + let (start, end) = selected_text_range(&text.to_string(), *index, *length); + if start != end { + let _ = text.unmark_utf16(start..end, key); + } + } + Command::MovableInsert { peer, index, value } => { + let list = self.doc(*peer).get_movable_list("movable"); + let _ = list.insert(modulo(*index, list.len() + 1), loro_value(value)); + } + Command::MovableDelete { + peer, + index, + length, + } => { + let list = self.doc(*peer).get_movable_list("movable"); + if !list.is_empty() { + let index = modulo(*index, list.len()); + let length = 1 + modulo(*length, list.len() - index); + let _ = list.delete(index, length); + } + } + Command::MovableSet { peer, index, value } => { + let list = self.doc(*peer).get_movable_list("movable"); + if !list.is_empty() { + let index = modulo(*index, list.len()); + let current = list + .get(index) + .map(|current| current.get_deep_value().to_json_value()); + if current.as_ref() != Some(value) { + let _ = list.set(index, loro_value(value)); + } + } + } + Command::MovableMove { peer, from, to } => { + let list = self.doc(*peer).get_movable_list("movable"); + if list.len() >= 2 { + let _ = list.mov(modulo(*from, list.len()), modulo(*to, list.len())); + } + } + Command::CounterIncrement { peer, delta } => { + if *delta != 0 { + let _ = self + .doc(*peer) + .get_counter("counter") + .increment(*delta as f64); + } + } + Command::TreeCreate { + peer, + parent, + value, + } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + let parent = parent + .and_then(|raw| nodes.get(modulo(raw, nodes.len())).copied()) + .map(TreeParentId::Node) + .unwrap_or(TreeParentId::Root); + if let Ok(node) = tree.create(parent) { + if let Ok(meta) = tree.get_meta(node) { + let _ = meta.insert("value", loro_value(value)); + } + } + } + Command::TreeMetaSet { + peer, + node, + key, + value, + } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + if let Some(node) = nodes.get(modulo(*node, nodes.len())).copied() { + if let Ok(meta) = tree.get_meta(node) { + let _ = meta.insert(key, loro_value(value)); + } + } + } + Command::TreeMove { peer, node, parent } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + if let Some(target) = nodes.get(modulo(*node, nodes.len())).copied() { + let candidates = nodes + .iter() + .copied() + .filter(|candidate| { + candidate != &target && !is_tree_descendant(&tree, *candidate, target) + }) + .collect::>(); + let selected_parent = parent + .and_then(|raw| candidates.get(modulo(raw, candidates.len())).copied()) + .map(TreeParentId::Node) + .unwrap_or(TreeParentId::Root); + if tree.parent(target).as_ref() != Some(&selected_parent) { + let _ = tree.mov(target, selected_parent); + } + } + } + Command::TreeDelete { peer, node } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + if let Some(node) = nodes.get(modulo(*node, nodes.len())).copied() { + let _ = tree.delete(node); + } + } + Command::Commit { peer, message } => { + let doc = self.doc(*peer); + doc.set_next_commit_message(&format!("interop-fuzz-{message}")); + doc.commit(); + } + Command::Enqueue { + source, + target, + slot, + mode, + } => self.enqueue(*source, *target, *slot, *mode), + Command::Deliver { slot, copies } => self.deliver(*slot, *copies), + Command::Save { peer, checkpoint } => { + let peer = peer_index(*peer, self.docs.len()); + self.checkpoints[peer][modulo(*checkpoint, SLOT_COUNT)] = + Some(self.docs[peer].state_frontiers()); + } + Command::Checkout { peer, checkpoint } => { + let peer = peer_index(*peer, self.docs.len()); + if let Some(frontiers) = + self.checkpoints[peer][modulo(*checkpoint, SLOT_COUNT)].as_ref() + { + if &self.docs[peer].state_frontiers() != frontiers { + let _ = self.docs[peer].checkout(frontiers); + } + } + } + Command::Attach { peer } => self.doc(*peer).attach(), + Command::Roundtrip { peer, mode } => { + let doc = self.doc(*peer); + let bytes = match mode { + TransportMode::Update => doc.export(ExportMode::all_updates()), + TransportMode::Snapshot => doc.export(ExportMode::Snapshot), + }; + if let Ok(bytes) = bytes { + let imported = LoroDoc::new(); + let _ = imported.import(&bytes); + } + } + } + } + + fn enqueue(&mut self, source: i64, target: i64, slot: i64, mode: TransportMode) { + let source = peer_index(source, self.docs.len()); + let mut target = peer_index(target, self.docs.len()); + if source == target { + target = (target + 1) % self.docs.len(); + } + let own_blob = match mode { + TransportMode::Update => { + self.docs[source].export(ExportMode::updates(&self.docs[target].oplog_vv())) + } + TransportMode::Snapshot => self.docs[source].export(ExportMode::Snapshot), + }; + let Ok(own_blob) = own_blob else { + return; + }; + let blob = self + .external_blobs + .as_ref() + .and_then(|blobs| blobs.get(self.enqueue_index)) + .cloned() + .unwrap_or_else(|| own_blob.clone()); + self.transport_blobs.push(own_blob); + self.enqueue_index += 1; + self.slots[modulo(slot, SLOT_COUNT)] = Some(Slot { blob, target }); + } + + fn deliver(&mut self, slot: i64, copies: i64) { + let Some(slot) = self.slots[modulo(slot, SLOT_COUNT)].clone() else { + return; + }; + let copies = 1 + modulo(copies, 3); + for _ in 0..copies { + let _ = self.docs[slot.target].import(&slot.blob); + } + } + + fn doc(&self, peer: i64) -> &LoroDoc { + &self.docs[peer_index(peer, self.docs.len())] + } +} + +fn observe_doc(doc: &LoroDoc) -> Observation { + Observation { + json: doc.get_deep_value().to_json_value(), + deep_with_id: doc.get_deep_value_with_id().to_json_value(), + version: version_pairs(doc.state_vv()), + oplog_version: version_pairs(doc.oplog_vv()), + frontiers: frontier_strings(doc.state_frontiers()), + oplog_frontiers: frontier_strings(doc.oplog_frontiers()), + detached: doc.is_detached(), + shallow: doc.is_shallow(), + op_count: doc.len_ops(), + change_count: doc.len_changes(), + } +} + +fn version_pairs(version: VersionVector) -> Vec<(String, i32)> { + let mut pairs = version + .iter() + .map(|(peer, counter)| (peer.to_string(), *counter)) + .collect::>(); + pairs.sort_by(|left, right| left.0.cmp(&right.0)); + pairs +} + +fn frontier_strings(frontiers: Frontiers) -> Vec { + let mut values = frontiers + .iter() + .map(|id| format!("{}@{}", id.counter, id.peer)) + .collect::>(); + values.sort(); + values +} + +fn live_tree_nodes(tree: &loro::LoroTree) -> Vec { + let mut nodes = tree + .get_nodes(false) + .into_iter() + .map(|node| node.id) + .collect::>(); + nodes.sort_by_key(ToString::to_string); + nodes +} + +fn is_tree_descendant(tree: &loro::LoroTree, node: TreeID, ancestor: TreeID) -> bool { + let mut parent = tree.parent(node); + while let Some(TreeParentId::Node(parent_id)) = parent { + if parent_id == ancestor { + return true; + } + parent = tree.parent(parent_id); + } + false +} + +fn utf16_boundaries(value: &str) -> Vec { + let mut output = vec![0]; + let mut offset = 0; + for character in value.chars() { + offset += character.len_utf16(); + output.push(offset); + } + output +} + +fn selected_text_range(value: &str, raw_index: i64, raw_length: i64) -> (usize, usize) { + let boundaries = utf16_boundaries(value); + let start_index = modulo(raw_index, boundaries.len()); + let remaining = boundaries.len() - start_index - 1; + if remaining == 0 { + return (boundaries[start_index], boundaries[start_index]); + } + let scalar_length = 1 + modulo(raw_length, remaining); + ( + boundaries[start_index], + boundaries[start_index + scalar_length], + ) +} + +fn loro_value(value: &Value) -> LoroValue { + LoroValue::from(value.clone()) +} + +fn peer_index(value: i64, peer_count: usize) -> usize { + modulo(value, peer_count) +} + +fn modulo(value: i64, length: usize) -> usize { + if length == 0 { + return 0; + } + value.rem_euclid(length as i64) as usize +} diff --git a/loro-js/README.md b/loro-js/README.md index bcc8bcdc9..44669425e 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -110,13 +110,21 @@ pnpm exec tsgo -p tsconfig.json --noEmit pnpm lint pnpm build pnpm fixtures:rewrite +pnpm fuzz:interop ``` From the repository root, the Rust-side compatibility suite is: ```sh cargo test -p loro --test loro_js_interop +pnpm test-loro-js-interop ``` +The differential fuzz suite crosses Rust/WASM and pure TypeScript update and +snapshot blobs, compares state/history/events after each command, checks +malformed-import atomicity, and replays its permanent corpus through native +Rust. See [`fuzz/README.md`](./fuzz/README.md) for replay and strict-profile +details. + The format implementation follows the repository reference in [`docs/encoding.md`](https://github.com/loro-dev/loro/blob/main/docs/encoding.md). diff --git a/loro-js/fuzz/.gitignore b/loro-js/fuzz/.gitignore new file mode 100644 index 000000000..d4f588edf --- /dev/null +++ b/loro-js/fuzz/.gitignore @@ -0,0 +1 @@ +artifacts/ diff --git a/loro-js/fuzz/README.md b/loro-js/fuzz/README.md new file mode 100644 index 000000000..3d908e6b7 --- /dev/null +++ b/loro-js/fuzz/README.md @@ -0,0 +1,77 @@ +# Rust / TypeScript interoperability fuzzing + +This suite executes one JSON scenario against the Rust/WASM `loro-crdt` +binding and the pure TypeScript `loro-js` runtime. After every command it +compares canonical document state, deep values with container IDs, text delta, +tree state, version/frontier data, pending/history counts, and normalized event +batches. + +Transport commands deliberately cross the implementations: Rust imports the +TypeScript blob and TypeScript imports the Rust blob. Both update and snapshot +formats are covered, along with duplicate and out-of-order delivery, checkout, +attach, and fresh-document round trips. A separate native Rust driver replays +the checked-in corpus in both directions so the oracle does not depend only on +the WASM binding. Deterministically corrupted update/snapshot inputs also check +matching acceptance or rejection and failure atomicity. + +## Run + +Build the Rust/WASM package once, then run the normal local profile: + +```sh +pnpm release-wasm +pnpm --filter loro-js fuzz:interop +``` + +The CI profile runs 500 traces of at most 60 commands. The native profile runs +the permanent corpus through `crates/fuzz/src/bin/loro_js_interop_driver.rs`. + +```sh +pnpm --filter loro-js fuzz:interop:ci +pnpm --filter loro-js fuzz:interop:native +``` + +Useful environment variables are: + +- `LORO_INTEROP_FUZZ_RUNS` +- `LORO_INTEROP_FUZZ_MAX_COMMANDS` +- `LORO_INTEROP_FUZZ_TIME_MS` +- `LORO_INTEROP_FUZZ_SEED` +- `LORO_INTEROP_FUZZ_PATH` + +Fast-check writes the shrunk scenario to `fuzz/artifacts/` and reports its +seed/path. Replay any scenario directly: + +```sh +node loro-js/fuzz/run-interop-fuzz.mjs --replay /absolute/path/to/failure.json +``` + +Promote useful minimized failures into `fuzz/corpus/`. + +## Profiles and oracle boundaries + +The default `stable` profile targets the interoperability surface currently +claimed by `loro-js`. It chooses valid UTF-16 boundaries and effective +map/tree/movable-list operations so a failure is about CRDT behavior rather +than whether an implementation records an invalid or semantic no-op call. +Encoded byte lengths and layouts are never compared because Loro encoding is +not canonical. + +Equivalent event representations are normalized: adjacent delta fragments are +coalesced, insert/delete operations at one cursor are put in a stable order, +and transaction-local tree create/delete pairs are treated as a net no-op. +Snapshot imports into a non-empty document still have known event differences, +so the stable profile compares their resulting state but not their event list. + +The `strict` profile adds rich-text mark/unmark generation, the corpus under +`fuzz/corpus/strict/`, and exact snapshot-import event comparison: + +```sh +pnpm --filter loro-js fuzz:interop:strict -- --corpus-only +``` + +It currently reproduces the documented rich-text-anchor limitation: after a +mark, Rust encodes later text insert positions in the anchor-bearing entity +coordinate space while the pure TypeScript runtime uses visible Unicode +positions. Keep the strict corpus failing until that implementation gap is +fixed; it is intentionally not part of the green CI gate. diff --git a/loro-js/fuzz/corpus/core.json b/loro-js/fuzz/corpus/core.json new file mode 100644 index 000000000..975cf3a13 --- /dev/null +++ b/loro-js/fuzz/corpus/core.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "peerCount": 3, + "commands": [ + { "kind": "map-set", "peer": 0, "key": "title", "value": "hello" }, + { "kind": "list-insert", "peer": 0, "index": 0, "value": "A" }, + { "kind": "text-insert", "peer": 0, "index": 0, "text": "a😀文" }, + { "kind": "movable-insert", "peer": 0, "index": 0, "value": "card-a" }, + { "kind": "movable-insert", "peer": 0, "index": 1, "value": "card-b" }, + { "kind": "movable-move", "peer": 0, "from": 0, "to": 1 }, + { "kind": "counter-increment", "peer": 0, "delta": 3 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": "root" }, + { "kind": "tree-create", "peer": 0, "parent": 0, "value": "child" }, + { "kind": "commit", "peer": 0, "message": 1 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "deliver", "slot": 0, "copies": 2 }, + { "kind": "map-set", "peer": 1, "key": "title", "value": "from peer 1" }, + { "kind": "text-insert", "peer": 1, "index": 2, "text": "!" }, + { "kind": "commit", "peer": 1, "message": 2 }, + { "kind": "enqueue", "source": 1, "target": 0, "slot": 1, "mode": "snapshot" }, + { "kind": "deliver", "slot": 1, "copies": 0 }, + { + "kind": "text-mark", + "peer": 0, + "index": 1, + "length": 0, + "key": "bold", + "value": true + }, + { "kind": "roundtrip", "peer": 0, "mode": "update" }, + { "kind": "roundtrip", "peer": 1, "mode": "snapshot" } + ] +} diff --git a/loro-js/fuzz/corpus/out-of-order.json b/loro-js/fuzz/corpus/out-of-order.json new file mode 100644 index 000000000..b5d2c2acb --- /dev/null +++ b/loro-js/fuzz/corpus/out-of-order.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "text-insert", "peer": 0, "index": 0, "text": "first" }, + { "kind": "commit", "peer": 0, "message": 1 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "text-insert", "peer": 0, "index": 5, "text": " second" }, + { "kind": "commit", "peer": 0, "message": 2 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 1, "mode": "update" }, + { "kind": "deliver", "slot": 1, "copies": 0 }, + { "kind": "deliver", "slot": 0, "copies": 1 }, + { "kind": "save", "peer": 1, "checkpoint": 0 }, + { "kind": "list-insert", "peer": 1, "index": 0, "value": "local" }, + { "kind": "commit", "peer": 1, "message": 3 }, + { "kind": "checkout", "peer": 1, "checkpoint": 0 }, + { "kind": "attach", "peer": 1 }, + { "kind": "roundtrip", "peer": 1, "mode": "snapshot" } + ] +} diff --git a/loro-js/fuzz/corpus/strict/richtext-anchor.json b/loro-js/fuzz/corpus/strict/richtext-anchor.json new file mode 100644 index 000000000..7b185fc98 --- /dev/null +++ b/loro-js/fuzz/corpus/strict/richtext-anchor.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "text-insert", "peer": 0, "index": 0, "text": "a😀文" }, + { + "kind": "text-mark", + "peer": 0, + "index": 1, + "length": 0, + "key": "bold", + "value": true + }, + { "kind": "commit", "peer": 0, "message": 1 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "deliver", "slot": 0, "copies": 0 }, + { "kind": "text-insert", "peer": 1, "index": 2, "text": "!" }, + { "kind": "commit", "peer": 1, "message": 2 }, + { "kind": "enqueue", "source": 1, "target": 0, "slot": 1, "mode": "update" }, + { "kind": "deliver", "slot": 1, "copies": 0 } + ] +} diff --git a/loro-js/fuzz/run-interop-fuzz.mjs b/loro-js/fuzz/run-interop-fuzz.mjs new file mode 100644 index 000000000..98b11ab39 --- /dev/null +++ b/loro-js/fuzz/run-interop-fuzz.mjs @@ -0,0 +1,182 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import fc from "fast-check"; + +import { + canonicalize, + runDifferentialScenario, + runMalformedImportChecks, + runSingleScenario, +} from "./runner.mjs"; +import { assertScenario, scenarioArbitrary } from "./scenario.mjs"; + +const fuzzRoot = path.dirname(fileURLToPath(import.meta.url)); +const packageRoot = path.resolve(fuzzRoot, ".."); +const workspaceRoot = path.resolve(packageRoot, ".."); +const jsEntry = path.join(packageRoot, "dist/index.js"); +const wasmEntries = [ + path.join(workspaceRoot, "crates/loro-wasm/nodejs/index.js"), + path.join(workspaceRoot, "crates/loro-wasm/nodejs/loro_wasm.js"), +]; +const corpusRoot = path.join(fuzzRoot, "corpus"); +const strictCorpusRoot = path.join(corpusRoot, "strict"); +const artifactsRoot = path.join(fuzzRoot, "artifacts"); + +const options = parseArgs(process.argv.slice(2)); +requireBuild(jsEntry, "pnpm --filter loro-js build"); +const wasmEntry = wasmEntries.find(existsSync); +if (wasmEntry === undefined) { + throw new Error(`missing Rust/WASM package; run pnpm release-wasm first`); +} + +const jsEngine = await import(pathToFileURL(jsEntry).href); +const importedWasmEngine = await import(pathToFileURL(wasmEntry).href); +const wasmEngine = + importedWasmEngine.LoroDoc === undefined + ? importedWasmEngine.default + : importedWasmEngine; +const corpus = readCorpus(options.replay, options.strict); +const malformedCases = runMalformedImportChecks(wasmEngine, jsEngine); + +for (const { name, scenario } of corpus) { + runDifferentialScenario(wasmEngine, jsEngine, scenario, { + strict: options.strict, + }); + if (process.env.LORO_INTEROP_NATIVE_DRIVER !== undefined) { + verifyNative(name, scenario, jsEngine, process.env.LORO_INTEROP_NATIVE_DRIVER); + } +} + +if (!options.corpusOnly) { + const arbitrary = scenarioArbitrary(options.maxCommands, { strict: options.strict }); + const property = fc.property(arbitrary, (scenario) => { + runDifferentialScenario(wasmEngine, jsEngine, scenario, { + strict: options.strict, + }); + }); + const check = fc.check(property, { + numRuns: options.runs, + seed: options.seed, + path: options.path, + endOnFailure: true, + interruptAfterTimeLimit: options.timeLimit, + }); + if (check.failed) { + mkdirSync(artifactsRoot, { recursive: true }); + const scenario = check.counterexample?.[0]; + const artifact = path.join(artifactsRoot, `failure-${Date.now()}.json`); + if (scenario !== undefined) { + writeFileSync(artifact, `${JSON.stringify(scenario, null, 2)}\n`); + } + const details = [ + `Loro interop fuzz failed after ${check.numRuns} runs`, + `seed=${check.seed}`, + `path=${check.counterexamplePath ?? ""}`, + `artifact=${artifact}`, + check.errorInstance?.stack ?? check.error ?? "unknown error", + ].join("\n"); + throw new Error(details); + } +} + +process.stdout.write( + `loro interop fuzz passed: profile=${options.strict ? "strict" : "stable"} corpus=${corpus.length} malformed=${malformedCases} random=${options.corpusOnly ? 0 : options.runs}\n`, +); + +function parseArgs(args) { + const ci = args.includes("--ci"); + const corpusOnly = args.includes("--corpus-only"); + const strict = args.includes("--strict"); + const replayIndex = args.indexOf("--replay"); + const replay = replayIndex >= 0 ? args[replayIndex + 1] : undefined; + const envRuns = Number.parseInt(process.env.LORO_INTEROP_FUZZ_RUNS ?? "", 10); + const envCommands = Number.parseInt( + process.env.LORO_INTEROP_FUZZ_MAX_COMMANDS ?? "", + 10, + ); + const envSeed = Number.parseInt(process.env.LORO_INTEROP_FUZZ_SEED ?? "", 10); + const envTime = Number.parseInt(process.env.LORO_INTEROP_FUZZ_TIME_MS ?? "", 10); + return { + ci, + corpusOnly, + strict, + replay, + runs: Number.isSafeInteger(envRuns) ? envRuns : ci ? 500 : 100, + maxCommands: Number.isSafeInteger(envCommands) ? envCommands : ci ? 60 : 40, + seed: Number.isSafeInteger(envSeed) ? envSeed : undefined, + path: process.env.LORO_INTEROP_FUZZ_PATH, + timeLimit: Number.isSafeInteger(envTime) ? envTime : ci ? 60_000 : undefined, + }; +} + +function requireBuild(entry, command) { + if (!existsSync(entry)) { + throw new Error(`missing ${entry}; run ${command} first`); + } +} + +function readCorpus(replay, strict) { + const files = + replay === undefined + ? [...jsonFiles(corpusRoot), ...(strict ? jsonFiles(strictCorpusRoot) : [])] + : [path.resolve(replay)]; + return files.map((file) => ({ + name: path.basename(file), + scenario: assertScenario(JSON.parse(readFileSync(file, "utf8"))), + })); +} + +function jsonFiles(root) { + return readdirSync(root) + .filter((name) => name.endsWith(".json")) + .sort() + .map((name) => path.join(root, name)); +} + +function verifyNative(name, scenario, js, driver) { + const jsSelf = runSingleScenario(js, scenario); + const nativeSelf = callNative(driver, { scenario }); + compareNative(`${name}: native self`, nativeSelf.observations, jsSelf.observations); + + const nativeFromJs = callNative(driver, { + scenario, + externalBlobs: jsSelf.transportBlobs, + }); + compareNative( + `${name}: JS -> native Rust`, + nativeFromJs.observations, + jsSelf.observations, + ); + + const jsFromNative = runSingleScenario(js, scenario, nativeSelf.transportBlobs); + compareNative( + `${name}: native Rust -> JS`, + jsFromNative.observations, + nativeSelf.observations, + ); +} + +function callNative(driver, input) { + const result = spawnSync(driver, [], { + input: JSON.stringify(input), + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`native driver failed: ${result.stderr || result.stdout}`); + } + return JSON.parse(result.stdout); +} + +function compareNative(label, left, right) { + const canonicalLeft = canonicalize(left); + const canonicalRight = canonicalize(right); + if (JSON.stringify(canonicalLeft) !== JSON.stringify(canonicalRight)) { + throw new Error( + `${label} mismatch\nnative=${JSON.stringify(canonicalLeft)}\njs=${JSON.stringify(canonicalRight)}`, + ); + } +} diff --git a/loro-js/fuzz/runner.mjs b/loro-js/fuzz/runner.mjs new file mode 100644 index 000000000..722086357 --- /dev/null +++ b/loro-js/fuzz/runner.mjs @@ -0,0 +1,796 @@ +import assert from "node:assert/strict"; +import { isDeepStrictEqual } from "node:util"; + +export function canonicalize(value) { + if (value === undefined) return { $undefined: true }; + if (typeof value === "number") { + if (Number.isNaN(value)) return { $number: "NaN" }; + if (value === Number.POSITIVE_INFINITY) return { $number: "+Infinity" }; + if (value === Number.NEGATIVE_INFINITY) return { $number: "-Infinity" }; + if (Object.is(value, -0)) return { $number: "-0" }; + return value; + } + if (typeof value === "bigint") return { $bigint: value.toString() }; + if (value instanceof Uint8Array) { + return { $bytes: Buffer.from(value).toString("base64") }; + } + if (value instanceof Map) { + return { + $map: [...value] + .map(([key, child]) => [String(key), canonicalize(child)]) + .sort(([left], [right]) => left.localeCompare(right)), + }; + } + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value === "string" && value.startsWith("idx:")) { + const marker = value.indexOf(", id:cid:"); + if (marker >= 0) return value.slice(marker + ", id:".length); + } + if (value === null || typeof value !== "object") return value; + + const output = {}; + for (const inputKey of Object.keys(value).sort()) { + const outputKey = inputKey === "fractionalIndex" ? "fractional_index" : inputKey; + const child = value[inputKey]; + if (child !== undefined) output[outputKey] = canonicalize(child); + } + if ("fractional_index" in output && !("parent" in output)) output.parent = null; + return output; +} + +function canonicalVersion(version) { + const json = typeof version?.toJSON === "function" ? version.toJSON() : version; + if (json instanceof Map) { + return [...json] + .map(([peer, counter]) => [String(peer), counter]) + .sort(([left], [right]) => left.localeCompare(right)); + } + return canonicalize(json); +} + +function canonicalFrontiers(frontiers) { + return frontiers + .map(({ peer, counter }) => `${counter}@${peer}`) + .sort((left, right) => left.localeCompare(right)); +} + +function normalizeDeepWithId(value, key) { + if (Array.isArray(value)) return value.map((child) => normalizeDeepWithId(child)); + if (value === null || typeof value !== "object") return value; + if (key === "meta" && typeof value.cid === "string" && Object.hasOwn(value, "value")) { + return normalizeDeepWithId(value.value, key); + } + return Object.fromEntries( + Object.entries(value).map(([childKey, child]) => [ + childKey, + normalizeDeepWithId(child, childKey), + ]), + ); +} + +function normalizeEventBatch(batch, doc) { + const normalized = canonicalize(batch); + // `origin` is binding metadata rather than a CRDT diff. The pure TS runtime + // does not currently expose every synthetic Rust origin (for example + // "checkout"), so compare the observable transition itself. + delete normalized.origin; + for (const key of ["from", "to"]) { + if (Array.isArray(normalized[key])) { + normalized[key].sort((left, right) => + `${left.counter}@${left.peer}`.localeCompare(`${right.counter}@${right.peer}`), + ); + } + } + if (Array.isArray(normalized.events)) { + for (const event of normalized.events) { + if (Array.isArray(event.diff?.diff)) { + event.diff.diff = canonicalizeDeltaOrder(coalesceDelta(event.diff.diff)); + } + if (event.diff?.type === "tree" && Array.isArray(event.diff.diff)) { + event.diff.diff = removeTransientTreeActions(event.diff.diff); + } + } + normalized.events = normalized.events.filter( + (event) => eventHasEffect(event) && !targetsDeletedTreeNode(event, doc), + ); + normalized.events.sort((left, right) => + JSON.stringify([left.target, left.path]).localeCompare( + JSON.stringify([right.target, right.path]), + ), + ); + } + return canonicalize(normalized); +} + +function canonicalizeDeltaOrder(delta) { + const output = []; + let group = []; + const flush = () => { + if (group.length === 0) return; + const deletes = group.reduce((sum, operation) => sum + (operation.delete ?? 0), 0); + if (deletes > 0) output.push({ delete: deletes }); + output.push( + ...coalesceDelta(group.filter((operation) => operation.delete === undefined)), + ); + group = []; + }; + for (const operation of delta) { + if (operation.retain !== undefined) { + flush(); + output.push(operation); + } else { + group.push(operation); + } + } + flush(); + return coalesceDelta(output); +} + +function removeTransientTreeActions(actions) { + const transientTargets = new Set(); + for (const action of actions) { + if (action.action !== "delete") continue; + if ( + actions.some( + (candidate) => + candidate.target === action.target && candidate.action === "create", + ) + ) { + transientTargets.add(action.target); + } + } + return actions.filter((action) => !transientTargets.has(action.target)); +} + +function targetsDeletedTreeNode(event, doc) { + const [root, node] = event.path ?? []; + return ( + typeof root === "string" && + typeof node === "string" && + node.includes("@") && + doc.getTree(root).isNodeDeleted(node) + ); +} + +function eventHasEffect(event) { + const diff = event.diff; + if (Array.isArray(diff?.diff)) return diff.diff.length > 0; + if (diff?.type === "map") return Object.keys(diff.updated ?? {}).length > 0; + if (diff?.type === "counter") return diff.increment !== 0; + return true; +} + +function coalesceDelta(delta) { + const output = []; + for (const operation of delta) { + const previous = output.at(-1); + if ( + previous !== undefined && + Object.hasOwn(previous, "insert") && + Object.hasOwn(operation, "insert") && + isDeepStrictEqual(previous.attributes, operation.attributes) + ) { + if (Array.isArray(previous.insert) && Array.isArray(operation.insert)) { + previous.insert.push(...operation.insert); + continue; + } + if (typeof previous.insert === "string" && typeof operation.insert === "string") { + previous.insert += operation.insert; + continue; + } + } + if ( + previous !== undefined && + typeof previous.delete === "number" && + typeof operation.delete === "number" + ) { + previous.delete += operation.delete; + continue; + } + if ( + previous !== undefined && + typeof previous.retain === "number" && + typeof operation.retain === "number" && + isDeepStrictEqual(previous.attributes, operation.attributes) + ) { + previous.retain += operation.retain; + continue; + } + output.push(structuredClone(operation)); + } + return output; +} + +export function observeDoc(doc) { + const pendingLength = doc.getPendingTxnLength(); + const tree = doc.getTree("tree"); + const treeNodes = tree + .getNodes({ withDeleted: true }) + .map((node) => node.toJSON()) + .sort((left, right) => String(left.id).localeCompare(String(right.id))); + const observation = { + json: doc.toJSON(), + deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), + textDelta: doc.getText("text").toDelta(), + treeNodes, + version: canonicalVersion(doc.version()), + frontiers: canonicalFrontiers(doc.frontiers()), + detached: doc.isDetached(), + shallow: doc.isShallow(), + pendingLength, + }; + if (pendingLength === 0) { + observation.oplogVersion = canonicalVersion(doc.oplogVersion()); + observation.oplogFrontiers = canonicalFrontiers(doc.oplogFrontiers()); + observation.opCount = doc.opCount(); + observation.changeCount = doc.changeCount(); + } + return canonicalize(observation); +} + +export function observeDocForNative(doc) { + return canonicalize({ + json: doc.toJSON(), + deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), + version: canonicalVersion(doc.version()), + oplogVersion: canonicalVersion(doc.oplogVersion()), + frontiers: canonicalFrontiers(doc.frontiers()), + oplogFrontiers: canonicalFrontiers(doc.oplogFrontiers()), + detached: doc.isDetached(), + shallow: doc.isShallow(), + opCount: doc.opCount(), + changeCount: doc.changeCount(), + }); +} + +function errorCategory(error) { + const message = String(error instanceof Error ? error.message : error).toLowerCase(); + if (message.includes("detached")) return "detached"; + if (message.includes("cycle") || message.includes("descendant")) return "tree-cycle"; + if ( + message.includes("range") || + message.includes("position") || + message.includes("index") + ) { + return "range"; + } + if (message.includes("shallow")) return "shallow"; + if (message.includes("checksum")) return "checksum"; + if (message.includes("dependency") || message.includes("pending")) return "dependency"; + return error?.constructor?.name ?? "Error"; +} + +function capture(action) { + try { + const value = action(); + return { ok: true, value: canonicalize(value) }; + } catch (error) { + return { ok: false, error: errorCategory(error) }; + } +} + +function modulo(value, length) { + if (length <= 0) return 0; + return ((value % length) + length) % length; +} + +function peerIndex(value, peerCount) { + return modulo(value, peerCount); +} + +function utf16Boundaries(value) { + const output = [0]; + for (let index = 0; index < value.length; ) { + const codePoint = value.codePointAt(index); + index += codePoint !== undefined && codePoint > 0xffff ? 2 : 1; + output.push(index); + } + return output; +} + +function selectedTextRange(text, rawIndex, rawLength, allowEmpty) { + const boundaries = utf16Boundaries(text.toString()); + const startIndex = modulo(rawIndex, boundaries.length); + const remaining = boundaries.length - startIndex - 1; + if (remaining <= 0) { + return { start: boundaries[startIndex], end: boundaries[startIndex] }; + } + const scalarLength = allowEmpty + ? modulo(rawLength, remaining + 1) + : 1 + modulo(rawLength, remaining); + return { + start: boundaries[startIndex], + end: boundaries[startIndex + scalarLength], + }; +} + +function liveTreeNodes(tree) { + return tree + .getNodes() + .slice() + .sort((left, right) => String(left.id).localeCompare(String(right.id))); +} + +function isTreeDescendant(node, ancestorId) { + for (let parent = node.parent(); parent !== undefined; parent = parent.parent()) { + if (parent.id === ancestorId) return true; + } + return false; +} + +function applyEdit(doc, command) { + switch (command.kind) { + case "map-set": + return doc.getMap("map").set(command.key, command.value); + case "map-delete": { + const map = doc.getMap("map"); + const keys = map.keys(); + if (keys.length === 0) return undefined; + const key = keys.includes(command.key) ? command.key : keys.sort()[0]; + return map.delete(key); + } + case "list-insert": { + const list = doc.getList("list"); + return list.insert(modulo(command.index, list.length + 1), command.value); + } + case "list-delete": { + const list = doc.getList("list"); + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + const length = 1 + modulo(command.length, list.length - index); + return list.delete(index, length); + } + case "text-insert": { + const text = doc.getText("text"); + const boundaries = utf16Boundaries(text.toString()); + return text.insert( + boundaries[modulo(command.index, boundaries.length)], + command.text, + ); + } + case "text-delete": { + const text = doc.getText("text"); + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.delete(start, end - start); + } + case "text-mark": { + const text = doc.getText("text"); + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.mark({ start, end }, command.key, command.value); + } + case "text-unmark": { + const text = doc.getText("text"); + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.unmark({ start, end }, command.key); + } + case "movable-insert": { + const list = doc.getMovableList("movable"); + return list.insert(modulo(command.index, list.length + 1), command.value); + } + case "movable-delete": { + const list = doc.getMovableList("movable"); + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + const length = 1 + modulo(command.length, list.length - index); + return list.delete(index, length); + } + case "movable-set": { + const list = doc.getMovableList("movable"); + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + if (isDeepStrictEqual(canonicalize(list.get(index)), canonicalize(command.value))) { + return undefined; + } + return list.set(index, command.value); + } + case "movable-move": { + const list = doc.getMovableList("movable"); + if (list.length < 2) return undefined; + return list.move( + modulo(command.from, list.length), + modulo(command.to, list.length), + ); + } + case "counter-increment": + return command.delta === 0 + ? undefined + : doc.getCounter("counter").increment(command.delta); + case "tree-create": { + const tree = doc.getTree("tree"); + const nodes = liveTreeNodes(tree); + const parent = + command.parent === null || nodes.length === 0 + ? undefined + : nodes[modulo(command.parent, nodes.length)].id; + const node = tree.createNode(parent); + node.data.set("value", command.value); + return node.id; + } + case "tree-meta-set": { + const nodes = liveTreeNodes(doc.getTree("tree")); + if (nodes.length === 0) return undefined; + return nodes[modulo(command.node, nodes.length)].data.set( + command.key, + command.value, + ); + } + case "tree-move": { + const tree = doc.getTree("tree"); + const nodes = liveTreeNodes(tree); + if (nodes.length === 0) return undefined; + const node = nodes[modulo(command.node, nodes.length)]; + const candidates = nodes.filter( + (candidate) => candidate.id !== node.id && !isTreeDescendant(candidate, node.id), + ); + const parent = + command.parent === null || candidates.length === 0 + ? undefined + : candidates[modulo(command.parent, candidates.length)]; + if (node.parent()?.id === parent?.id) return undefined; + return tree.move(node.id, parent?.id); + } + case "tree-delete": { + const tree = doc.getTree("tree"); + const nodes = liveTreeNodes(tree); + if (nodes.length === 0) return undefined; + return tree.delete(nodes[modulo(command.node, nodes.length)].id); + } + default: + throw new RangeError(`unsupported edit command ${command.kind}`); + } +} + +function createDoc(engine, peer, initializeRoots = true) { + const doc = new engine.LoroDoc(); + if (peer !== undefined) doc.setPeerId(peer + 1); + doc.setRecordTimestamp(false); + doc.setChangeMergeInterval(0); + if (initializeRoots) { + doc.getMap("map"); + doc.getList("list"); + doc.getText("text"); + doc.getMovableList("movable"); + doc.getCounter("counter"); + doc.getTree("tree").enableFractionalIndex(0); + } + return doc; +} + +function exportFor(doc, target, mode) { + return mode === "snapshot" + ? doc.export({ mode: "snapshot" }) + : doc.export({ mode: "update", from: target.oplogVersion() }); +} + +function compare(label, left, right) { + try { + assert.deepStrictEqual(left, right); + } catch (cause) { + const error = new Error( + `${label}\nleft=${JSON.stringify(left)}\nright=${JSON.stringify(right)}`, + ); + error.cause = cause; + throw error; + } +} + +class EngineWorld { + constructor(engine, scenario, externalBlobs) { + this.engine = engine; + this.docs = Array.from({ length: scenario.peerCount }, (_, peer) => + createDoc(engine, peer), + ); + this.slots = new Map(); + this.checkpoints = this.docs.map(() => new Map()); + this.externalBlobs = externalBlobs?.map((bytes) => new Uint8Array(bytes)); + this.transportBlobs = []; + this.enqueueIndex = 0; + this.eventQueues = this.docs.map(() => []); + this.subscriptions = this.docs.map((doc, peer) => + doc.subscribe((batch) => { + const normalized = normalizeEventBatch(batch, doc); + if (normalized.events.length > 0) this.eventQueues[peer].push(normalized); + }), + ); + } + + execute(command) { + const doc = this.docs[peerIndex(command.peer ?? 0, this.docs.length)]; + if (command.kind.includes("-") && isEditCommand(command.kind)) { + return capture(() => applyEdit(doc, command)); + } + switch (command.kind) { + case "commit": + return capture(() => doc.commit({ message: `interop-fuzz-${command.message}` })); + case "enqueue": { + const sourceIndex = peerIndex(command.source, this.docs.length); + let targetIndex = peerIndex(command.target, this.docs.length); + if (targetIndex === sourceIndex) + targetIndex = (targetIndex + 1) % this.docs.length; + const ownBlob = exportFor( + this.docs[sourceIndex], + this.docs[targetIndex], + command.mode, + ); + const external = this.externalBlobs?.[this.enqueueIndex]; + const blob = external ?? ownBlob; + this.transportBlobs.push([...ownBlob]); + this.enqueueIndex += 1; + this.slots.set(modulo(command.slot, 8), { + blob, + targetIndex, + mode: command.mode, + }); + return { ok: true, value: "enqueued" }; + } + case "deliver": { + const slot = this.slots.get(modulo(command.slot, 8)); + if (slot === undefined) return { ok: true, value: "empty-slot" }; + const copies = 1 + modulo(command.copies, 3); + const statuses = []; + for (let copy = 0; copy < copies; copy += 1) { + statuses.push(canonicalize(this.docs[slot.targetIndex].import(slot.blob))); + } + return { ok: true, value: statuses }; + } + case "save": + this.checkpoints[peerIndex(command.peer, this.docs.length)].set( + modulo(command.checkpoint, 8), + doc.frontiers(), + ); + return { ok: true, value: undefined }; + case "checkout": { + const frontiers = this.checkpoints[peerIndex(command.peer, this.docs.length)].get( + modulo(command.checkpoint, 8), + ); + if (frontiers === undefined) return { ok: true, value: "missing-checkpoint" }; + if ( + isDeepStrictEqual( + canonicalFrontiers(frontiers), + canonicalFrontiers(doc.frontiers()), + ) + ) { + return { ok: true, value: "current-checkpoint" }; + } + return capture(() => doc.checkout(frontiers)); + } + case "attach": + return capture(() => doc.attach()); + case "roundtrip": + return this.roundtrip(doc, command.mode); + default: + throw new RangeError(`unsupported command ${command.kind}`); + } + } + + roundtrip(doc, mode) { + return capture(() => { + const bytes = + mode === "snapshot" + ? doc.export({ mode: "snapshot" }) + : doc.export({ mode: "update" }); + const imported = createDoc(this.engine, undefined, false); + imported.import(bytes); + return observeDoc(imported); + }); + } + + finalNativeObservations() { + return this.docs.map(observeDocForNative); + } + + drainEvents() { + this.engine.callPendingEvents?.(); + return this.eventQueues.map((queue) => queue.splice(0)); + } +} + +function isEditCommand(kind) { + return ( + kind.startsWith("map-") || + kind.startsWith("list-") || + kind.startsWith("text-") || + kind.startsWith("movable-") || + kind.startsWith("counter-") || + kind.startsWith("tree-") + ); +} + +export function runSingleScenario(engine, scenario, externalBlobs) { + const world = new EngineWorld(engine, scenario, externalBlobs); + const results = []; + for (const command of scenario.commands) results.push(world.execute(command)); + return { + observations: world.finalNativeObservations(), + transportBlobs: world.transportBlobs, + results: canonicalize(results), + }; +} + +export function runMalformedImportChecks(wasmEngine, jsEngine) { + const engines = [ + ["wasm", wasmEngine], + ["js", jsEngine], + ]; + let cases = 0; + for (const [producerName, producer] of engines) { + const source = createDoc(producer, 0); + source.getMap("map").set("payload", { nested: [1, true, "😀"] }); + source.getText("text").insert(0, "malformed 😀 文"); + source.getList("list").push("item"); + source.commit({ message: "malformed-seed" }); + for (const mode of ["update", "snapshot"]) { + const blob = source.export({ mode }); + for (const [mutation, bytes] of malformedVariants(blob)) { + const outcomes = engines.map(([consumerName, engine]) => [ + consumerName, + malformedOutcome(engine, bytes), + ]); + compare( + `malformed import mismatch: producer=${producerName} mode=${mode} mutation=${mutation}`, + outcomes[0][1], + outcomes[1][1], + ); + cases += 1; + } + } + } + return cases; +} + +function malformedVariants(blob) { + const half = Math.max(1, Math.floor(blob.length / 2)); + return [ + ["empty", new Uint8Array()], + ["prefix-1", blob.slice(0, 1)], + ["prefix-half", blob.slice(0, half)], + ["truncate-last", blob.slice(0, -1)], + ["append-garbage", Uint8Array.from([...blob, 0xff])], + ["flip-header", flippedByte(blob, 0)], + ["flip-middle", flippedByte(blob, Math.floor(blob.length / 2))], + ["flip-tail", flippedByte(blob, blob.length - 1)], + ]; +} + +function flippedByte(blob, index) { + const output = blob.slice(); + output[index] ^= 0xff; + return output; +} + +function malformedOutcome(engine, bytes) { + const target = createDoc(engine, 97); + target.getMap("map").set("sentinel", "unchanged"); + target.getText("text").insert(0, "stable"); + target.commit({ message: "malformed-target" }); + const before = observeDoc(target); + try { + target.import(bytes); + return { accepted: true, observation: observeDoc(target) }; + } catch { + return { + accepted: false, + atomic: isDeepStrictEqual(before, observeDoc(target)), + }; + } +} + +export function runDifferentialScenario( + wasmEngine, + jsEngine, + scenario, + { strict = false } = {}, +) { + const wasm = new EngineWorld(wasmEngine, scenario); + const js = new EngineWorld(jsEngine, scenario); + + for (let index = 0; index < scenario.commands.length; index += 1) { + const command = scenario.commands[index]; + let wasmResult; + let jsResult; + let compareEvents = true; + if (command.kind === "deliver") { + const wasmSlot = wasm.slots.get(modulo(command.slot, 8)); + const jsSlot = js.slots.get(modulo(command.slot, 8)); + if (wasmSlot === undefined || jsSlot === undefined) { + wasmResult = { ok: true, value: "empty-slot" }; + jsResult = { ok: true, value: "empty-slot" }; + } else { + compareEvents = strict || wasmSlot.mode !== "snapshot"; + const copies = 1 + modulo(command.copies, 3); + const wasmStatuses = []; + const jsStatuses = []; + for (let copy = 0; copy < copies; copy += 1) { + wasmStatuses.push( + canonicalize(wasm.docs[wasmSlot.targetIndex].import(jsSlot.blob)), + ); + jsStatuses.push( + canonicalize(js.docs[jsSlot.targetIndex].import(wasmSlot.blob)), + ); + } + wasmResult = { ok: true, value: wasmStatuses }; + jsResult = { ok: true, value: jsStatuses }; + } + } else if (command.kind === "roundtrip") { + const peer = peerIndex(command.peer, wasm.docs.length); + wasmResult = crossRoundtrip( + wasmEngine, + jsEngine, + wasm.docs[peer], + js.docs[peer], + command.mode, + ); + jsResult = wasmResult; + } else { + wasmResult = wasm.execute(command); + jsResult = js.execute(command); + } + + compare( + `command result mismatch at ${index}: ${JSON.stringify(command)}`, + wasmResult, + jsResult, + ); + const wasmEvents = wasm.drainEvents(); + const jsEvents = js.drainEvents(); + if (compareEvents) { + compare( + `event mismatch at command ${index}: ${JSON.stringify(command)}`, + wasmEvents, + jsEvents, + ); + } + for (let peer = 0; peer < scenario.peerCount; peer += 1) { + compare( + `document mismatch at command ${index}, peer ${peer}: ${JSON.stringify(command)}`, + observeDoc(wasm.docs[peer]), + observeDoc(js.docs[peer]), + ); + } + } +} + +function crossRoundtrip(wasmEngine, jsEngine, wasmDoc, jsDoc, mode) { + return capture(() => { + const wasmBytes = + mode === "snapshot" + ? wasmDoc.export({ mode: "snapshot" }) + : wasmDoc.export({ mode: "update" }); + const jsBytes = + mode === "snapshot" + ? jsDoc.export({ mode: "snapshot" }) + : jsDoc.export({ mode: "update" }); + const variants = [ + [wasmEngine, wasmBytes], + [jsEngine, wasmBytes], + [wasmEngine, jsBytes], + [jsEngine, jsBytes], + ].map(([engine, bytes]) => { + const doc = createDoc(engine, undefined, false); + doc.import(bytes); + return observeDoc(doc); + }); + for (let index = 1; index < variants.length; index += 1) { + compare(`roundtrip variant ${index} mismatch`, variants[0], variants[index]); + } + return variants[0]; + }); +} diff --git a/loro-js/fuzz/scenario.mjs b/loro-js/fuzz/scenario.mjs new file mode 100644 index 000000000..9c97a71a8 --- /dev/null +++ b/loro-js/fuzz/scenario.mjs @@ -0,0 +1,157 @@ +import fc from "fast-check"; + +const alphabet = ["a", "b", "c", " ", "\n", "\0", "é", "文", "א", "😀", "𐐷", "\u0301"]; + +const smallString = (options = {}) => + fc + .array(fc.constantFrom(...alphabet), { + minLength: options.minLength ?? 0, + maxLength: options.maxLength ?? 8, + }) + .map((parts) => parts.join("")); + +const key = fc.constantFrom("a", "b", "title", "value", "emoji😀", "empty"); +const scalar = fc.oneof( + fc.constant(null), + fc.boolean(), + fc.integer({ min: -1_000, max: 1_000 }), + smallString({ maxLength: 6 }), +); +const value = fc.oneof( + scalar, + fc.array(scalar, { maxLength: 4 }), + fc.dictionary(key, scalar, { maxKeys: 4 }), +); +const byte = fc.integer({ min: 0, max: 255 }); +const peer = byte; +const slot = fc.integer({ min: 0, max: 7 }); + +const commandArbitraries = [ + fc.record({ kind: fc.constant("map-set"), peer, key, value }), + fc.record({ kind: fc.constant("map-delete"), peer, key }), + fc.record({ kind: fc.constant("list-insert"), peer, index: byte, value }), + fc.record({ + kind: fc.constant("list-delete"), + peer, + index: byte, + length: byte, + }), + fc.record({ + kind: fc.constant("text-insert"), + peer, + index: byte, + text: smallString({ minLength: 1, maxLength: 8 }), + }), + fc.record({ + kind: fc.constant("text-delete"), + peer, + index: byte, + length: byte, + }), + fc.record({ + kind: fc.constant("text-mark"), + peer, + index: byte, + length: byte, + key: fc.constantFrom("bold", "link", "comment"), + value: scalar, + }), + fc.record({ + kind: fc.constant("text-unmark"), + peer, + index: byte, + length: byte, + key: fc.constantFrom("bold", "link", "comment"), + }), + fc.record({ kind: fc.constant("movable-insert"), peer, index: byte, value }), + fc.record({ + kind: fc.constant("movable-delete"), + peer, + index: byte, + length: byte, + }), + fc.record({ kind: fc.constant("movable-set"), peer, index: byte, value }), + fc.record({ kind: fc.constant("movable-move"), peer, from: byte, to: byte }), + fc.record({ + kind: fc.constant("counter-increment"), + peer, + delta: fc.integer({ min: -20, max: 20 }), + }), + fc.record({ + kind: fc.constant("tree-create"), + peer, + parent: fc.option(byte, { nil: null }), + value, + }), + fc.record({ + kind: fc.constant("tree-meta-set"), + peer, + node: byte, + key, + value, + }), + fc.record({ + kind: fc.constant("tree-move"), + peer, + node: byte, + parent: fc.option(byte, { nil: null }), + }), + fc.record({ kind: fc.constant("tree-delete"), peer, node: byte }), + fc.record({ kind: fc.constant("commit"), peer, message: byte }), + fc.record({ + kind: fc.constant("enqueue"), + source: peer, + target: peer, + slot, + mode: fc.constantFrom("update", "snapshot"), + }), + fc.record({ kind: fc.constant("deliver"), slot, copies: byte }), + fc.record({ kind: fc.constant("save"), peer, checkpoint: slot }), + fc.record({ kind: fc.constant("checkout"), peer, checkpoint: slot }), + fc.record({ kind: fc.constant("attach"), peer }), + fc.record({ + kind: fc.constant("roundtrip"), + peer, + mode: fc.constantFrom("update", "snapshot"), + }), +]; + +// Rich-text anchors are represented differently by the pure TypeScript runtime +// today. Keep them in the strict profile so the known divergence remains easy +// to reproduce without making the stable compatibility gate permanently red. +const stableCommandArbitraries = commandArbitraries.filter( + (_, index) => index !== 6 && index !== 7, +); + +export function scenarioArbitrary(maxCommands, { strict = false } = {}) { + const commands = strict ? commandArbitraries : stableCommandArbitraries; + // Repeat edits and transport commands to bias generated traces toward states + // where multiple container kinds and peers interact. + const command = fc.oneof( + ...commands, + ...commands.slice(0, strict ? 18 : 16), + commandArbitraries[18], + commandArbitraries[19], + ); + return fc.record({ + schemaVersion: fc.constant(1), + peerCount: fc.integer({ min: 2, max: 4 }), + commands: fc.array(command, { minLength: 1, maxLength: maxCommands }), + }); +} + +export function assertScenario(value) { + if (value === null || typeof value !== "object") { + throw new TypeError("scenario must be an object"); + } + if (value.schemaVersion !== 1) { + throw new RangeError(`unsupported scenario schema ${String(value.schemaVersion)}`); + } + if (!Number.isSafeInteger(value.peerCount) || value.peerCount < 2) { + throw new RangeError("scenario peerCount must be at least two"); + } + if (!Array.isArray(value.commands)) { + throw new TypeError("scenario commands must be an array"); + } + return value; +} diff --git a/loro-js/package.json b/loro-js/package.json index 12ae6541f..b12295638 100644 --- a/loro-js/package.json +++ b/loro-js/package.json @@ -50,6 +50,10 @@ "build": "vp pack", "check": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm test && pnpm build", "clean": "rm -rf dist", + "fuzz:interop": "pnpm build && node fuzz/run-interop-fuzz.mjs", + "fuzz:interop:ci": "pnpm build && node fuzz/run-interop-fuzz.mjs --ci", + "fuzz:interop:native": "cargo build -p fuzz --bin loro_js_interop_driver && LORO_INTEROP_NATIVE_DRIVER=../target/debug/loro_js_interop_driver pnpm fuzz:interop -- --corpus-only", + "fuzz:interop:strict": "pnpm build && node fuzz/run-interop-fuzz.mjs --strict", "format": "vp fmt", "format:check": "vp fmt --check", "fixtures:rewrite": "pnpm build && node scripts/rewrite-rust-fixture.mjs", @@ -60,6 +64,7 @@ "devDependencies": { "@types/node": "^22.0.0", "@typescript/native-preview": "7.0.0-dev.20260506.1", + "fast-check": "^4.9.0", "typescript": "^5.6.3", "vite": "8.0.10", "vite-plus": "0.1.20", diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 3a7e5ac42..c5c2e64d7 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -6194,27 +6194,6 @@ function changeLength(change: DecodedChange): number { return length; } -function advanceUnicodeScalars(value: string, start: number, length: number): number { - let offset = start; - for (let index = 0; index < length; index += 1) { - if (offset >= value.length) { - throw new Error("text snapshot span exceeds the decoded text"); - } - const first = value.charCodeAt(offset); - offset += 1; - if ( - first >= 0xd800 && - first <= 0xdbff && - offset < value.length && - value.charCodeAt(offset) >= 0xdc00 && - value.charCodeAt(offset) <= 0xdfff - ) { - offset += 1; - } - } - return offset; -} - function validateStateSnapshotTextIds( store: StateSnapshotStore, ): ReadonlyMap { @@ -7961,12 +7940,7 @@ function splitsSurrogatePair(value: string, at: number): boolean { if (at <= 0 || at >= value.length) return false; const before = value.charCodeAt(at - 1); const after = value.charCodeAt(at); - return ( - before >= 0xd800 && - before <= 0xdbff && - after >= 0xdc00 && - after <= 0xdfff - ); + return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff; } function stringDelta(before: string, after: string): Delta[] { @@ -8091,7 +8065,9 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { if (left instanceof Uint8Array && right instanceof Uint8Array) { return bytesEqual(left, right); } + if (left instanceof Uint8Array || right instanceof Uint8Array) return false; if (isContainer(left) && isContainer(right)) return left.id === right.id; + if (isContainer(left) || isContainer(right)) return false; if (Array.isArray(left) && Array.isArray(right)) { if (left.length !== right.length) return false; for (let index = 0; index < left.length; index += 1) { @@ -8099,6 +8075,7 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { } return true; } + if (Array.isArray(left) || Array.isArray(right)) return false; if (left instanceof Map && right instanceof Map) { if (left.size !== right.size) return false; for (const [key, value] of left) { @@ -8106,6 +8083,7 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { } return true; } + if (left instanceof Map || right instanceof Map) return false; if ( typeof left === "object" && left !== null && @@ -8122,7 +8100,7 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { } } let rightCount = 0; - for (const key in rightRecord) rightCount += 1; + for (const _key in rightRecord) rightCount += 1; return leftCount === rightCount; } return false; diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 7b7902ba5..752d2b4be 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -1610,6 +1610,27 @@ describe("loro-wasm-compatible runtime", () => { expect(doc.opCount()).toBe(opCount); }); + test("does not confuse empty value shapes when detecting no-op edits", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + const movable = doc.getMovableList("movable"); + map.set("value", []); + movable.push([]); + doc.commit(); + const opCount = doc.opCount(); + + map.set("value", {}); + movable.set(0, {}); + doc.commit(); + map.set("value", new Uint8Array()); + movable.set(0, new Uint8Array()); + doc.commit(); + + expect(map.get("value")).toEqual(new Uint8Array()); + expect(movable.get(0)).toEqual(new Uint8Array()); + expect(doc.opCount()).toBe(opCount + 4); + }); + test("resurfaces preserved state and switches mergeable kinds", () => { const doc = new LoroDoc(); const root = doc.getMap("state"); diff --git a/package.json b/package.json index 7c8d85ff6..8ebba034f 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test-esbuild": "pnpm --dir examples/test-esbuild install --ignore-workspace --prefer-offline && pnpm --filter loro-test-esbuild serve", "test-bundlers": "pnpm --dir examples/bundler-smoke-tests run test:fast", "test-bundlers-next": "pnpm --dir examples/bundler-smoke-tests run test:next", + "test-loro-js-interop": "pnpm --filter loro-js fuzz:interop:ci && pnpm --filter loro-js fuzz:interop:native", "run-fuzz-corpus": "node ./scripts/cargo-fuzz-run.mjs all -- -max_total_time=1", "fix": "cargo clippy --fix --features=test_utils", "vet": "cargo vet", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 549f7b79a..79d3d2dfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -180,6 +180,9 @@ importers: '@typescript/native-preview': specifier: 7.0.0-dev.20260506.1 version: 7.0.0-dev.20260506.1 + fast-check: + specifier: ^4.9.0 + version: 4.9.0 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2466,6 +2469,10 @@ packages: resolution: {integrity: sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==} engines: {node: '>=4'} + fast-check@4.9.0: + resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -3326,6 +3333,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -6075,6 +6085,10 @@ snapshots: iconv-lite: 0.4.24 tmp: 0.0.33 + fast-check@4.9.0: + dependencies: + pure-rand: 8.4.2 + fast-deep-equal@3.1.3: {} fast-diff@1.1.2: {} @@ -6915,6 +6929,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + quansync@0.2.11: {} queue-microtask@1.2.3: {} From 27ec7f7aeb8c6d15b68fbceee5996468bfa30a29 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 20 Jul 2026 01:38:43 +0800 Subject: [PATCH 2/5] fix: align Rust and JS interop behavior Fix event frontier tracking, UTF-16 delete hints, tree snapshot and checkout semantics, and pure TypeScript event parity found by differential fuzzing. Promote minimized failures to the permanent corpus. --- .changeset/fuzzy-loros-agree.md | 5 +- context/loro-js-interop-fuzz.md | 14 + crates/examples/examples/b4_bench.rs | 10 +- crates/examples/examples/mergeable_notes.rs | 5 +- crates/fuzz/src/container/map.rs | 10 +- .../src/container/richtext/richtext_state.rs | 2 +- crates/loro-internal/src/handler.rs | 63 +- crates/loro-internal/src/state.rs | 21 +- crates/loro-internal/src/state/map_state.rs | 4 +- .../loro-internal/src/state/richtext_state.rs | 4 +- crates/loro-internal/src/state/tree_state.rs | 101 +- crates/loro-internal/src/txn.rs | 85 +- .../tests/mergeable_container/convergence.rs | 5 +- .../tests/mergeable_container/delete.rs | 5 +- .../tests/mergeable_container/snapshot.rs | 4 +- crates/loro-internal/tests/test.rs | 32 + crates/loro/tests/mergeable_public_api.rs | 10 +- loro-js/fuzz/README.md | 5 +- .../fuzz/corpus/checkout-pending-tree.json | 9 + .../checkout-tree-metadata-revival.json | 10 + .../corpus/checkout-tree-old-indices.json | 10 + .../fuzz/corpus/concurrent-tree-reindex.json | 10 + .../fuzz/corpus/deleted-tree-position.json | 28 + .../corpus/empty-import-event-frontier.json | 43 + .../fuzz/corpus/import-tree-replay-noop.json | 30 + .../local-map-delete-restore-event.json | 31 + .../fuzz/corpus/local-tree-nested-create.json | 10 + .../corpus/local-tree-parent-reindex.json | 11 + loro-js/fuzz/corpus/retain-only-event.json | 11 + .../corpus/snapshot-tree-sibling-order.json | 11 + .../fuzz/corpus/trailing-retain-event.json | 38 + .../fuzz/corpus/transient-tree-reindex.json | 30 + .../fuzz/corpus/transient-tree-subtree.json | 30 + loro-js/fuzz/corpus/transient-tree.json | 9 + .../fuzz/corpus/utf16-delete-hint-split.json | 29 + loro-js/fuzz/run-interop-fuzz.mjs | 47 +- loro-js/fuzz/runner.mjs | 877 ++++++++++++------ loro-js/src/runtime/containers.ts | 124 ++- loro-js/src/runtime/document.ts | 190 +++- loro-js/tests/event-diff.test.ts | 40 + loro-js/tests/runtime.test.ts | 116 ++- loro-js/tests/version-transition.test.ts | 69 ++ 42 files changed, 1706 insertions(+), 492 deletions(-) create mode 100644 loro-js/fuzz/corpus/checkout-pending-tree.json create mode 100644 loro-js/fuzz/corpus/checkout-tree-metadata-revival.json create mode 100644 loro-js/fuzz/corpus/checkout-tree-old-indices.json create mode 100644 loro-js/fuzz/corpus/concurrent-tree-reindex.json create mode 100644 loro-js/fuzz/corpus/deleted-tree-position.json create mode 100644 loro-js/fuzz/corpus/empty-import-event-frontier.json create mode 100644 loro-js/fuzz/corpus/import-tree-replay-noop.json create mode 100644 loro-js/fuzz/corpus/local-map-delete-restore-event.json create mode 100644 loro-js/fuzz/corpus/local-tree-nested-create.json create mode 100644 loro-js/fuzz/corpus/local-tree-parent-reindex.json create mode 100644 loro-js/fuzz/corpus/retain-only-event.json create mode 100644 loro-js/fuzz/corpus/snapshot-tree-sibling-order.json create mode 100644 loro-js/fuzz/corpus/trailing-retain-event.json create mode 100644 loro-js/fuzz/corpus/transient-tree-reindex.json create mode 100644 loro-js/fuzz/corpus/transient-tree-subtree.json create mode 100644 loro-js/fuzz/corpus/transient-tree.json create mode 100644 loro-js/fuzz/corpus/utf16-delete-hint-split.json diff --git a/.changeset/fuzzy-loros-agree.md b/.changeset/fuzzy-loros-agree.md index e679603f8..9b67a4b4f 100644 --- a/.changeset/fuzzy-loros-agree.md +++ b/.changeset/fuzzy-loros-agree.md @@ -3,4 +3,7 @@ --- Keep semantic no-op detection type-safe so empty arrays, objects, maps, binary -values, and containers cannot be mistaken for one another. +values, and containers cannot be mistaken for one another. Match Rust by +rejecting `isNodeDeleted()` queries for tree nodes that do not exist and by +preserving deleted-tree roots, descendants, positions, and last-move metadata +with the same observable semantics. diff --git a/context/loro-js-interop-fuzz.md b/context/loro-js-interop-fuzz.md index a1205feb5..a2679f429 100644 --- a/context/loro-js-interop-fuzz.md +++ b/context/loro-js-interop-fuzz.md @@ -25,6 +25,20 @@ into a non-empty document. The strict corpus records these known gaps. In particular, `fuzz/corpus/strict/richtext-anchor.json` demonstrates the remaining anchor/entity-position mismatch after a marked range receives a later insert. +Event normalization is payload-only. Subscription callbacks enqueue raw event +batches, which are normalized after the public command returns. Do not query the +live document from a callback: the WASM adapter flushes queued callbacks at the +outer JS method boundary while the pure TypeScript runtime emits during the +internal commit, so callback-time state can differ even when event payloads and +the resulting document state agree. + +Map event recording is operation-aware rather than only comparing the values at +the start and end of a batch. If a key visibly changes and is then restored in +the same local or imported change, Rust reports the final value in `updated`, so +the TypeScript runtime must retain that key. An incoming same-value winner that +never changes the visible value remains an event no-op. The regression scenario +is `fuzz/corpus/local-map-delete-restore-event.json`. + Run the complete green smoke from the repository root with: ```sh diff --git a/crates/examples/examples/b4_bench.rs b/crates/examples/examples/b4_bench.rs index e7f3d660f..9bd4fda9e 100644 --- a/crates/examples/examples/b4_bench.rs +++ b/crates/examples/examples/b4_bench.rs @@ -69,7 +69,10 @@ fn report() { let d = apply(&actions, 1); d.export(ExportMode::Snapshot).unwrap() }); - println!(" export(+apply,nocache):{:>8.2?} (includes a fresh apply)", t_export_nc); + println!( + " export(+apply,nocache):{:>8.2?} (includes a fresh apply)", + t_export_nc + ); // ---- Snapshot import ---- let mem_before = get_mem_usage(); @@ -89,7 +92,10 @@ fn report() { let v = d.get_deep_value(); std::hint::black_box(v); }); - println!(" import + toJSON: {:>10.2?} (forces full state materialization)", t_import_val); + println!( + " import + toJSON: {:>10.2?} (forces full state materialization)", + t_import_val + ); std::hint::black_box(&imported); // ---- B4 x100 ---- diff --git a/crates/examples/examples/mergeable_notes.rs b/crates/examples/examples/mergeable_notes.rs index 60f1a0ab6..44ffa8148 100644 --- a/crates/examples/examples/mergeable_notes.rs +++ b/crates/examples/examples/mergeable_notes.rs @@ -150,7 +150,10 @@ fn print_report(r: &Report) { println!(" import snapshot: {:?}", r.import_time); println!(" resident after import: {}", r.imported_resident); println!(" get_deep_value: {:?}", r.deep_value_time); - println!(" resident after deep_value: {}", r.after_deep_value_resident); + println!( + " resident after deep_value: {}", + r.after_deep_value_resident + ); } fn main() { diff --git a/crates/fuzz/src/container/map.rs b/crates/fuzz/src/container/map.rs index 84951da5d..b9e22da28 100644 --- a/crates/fuzz/src/container/map.rs +++ b/crates/fuzz/src/container/map.rs @@ -209,7 +209,9 @@ impl Actionable for MapAction { // Type-conflict rejection (`ArgErr`) is an expected outcome — swallow it // here rather than via `unwrap` so unrelated `ArgErr` sources still panic. let result = match kind { - ContainerType::Map => handler.ensure_mergeable_map(key).map(|m| m.to_container()), + ContainerType::Map => { + handler.ensure_mergeable_map(key).map(|m| m.to_container()) + } ContainerType::List => { handler.ensure_mergeable_list(key).map(|l| l.to_container()) } @@ -222,9 +224,9 @@ impl Actionable for MapAction { ContainerType::Tree => { handler.ensure_mergeable_tree(key).map(|t| t.to_container()) } - ContainerType::Counter => { - handler.ensure_mergeable_counter(key).map(|c| c.to_container()) - } + ContainerType::Counter => handler + .ensure_mergeable_counter(key) + .map(|c| c.to_container()), ContainerType::Unknown(_) => return None, }; match result { diff --git a/crates/loro-internal/src/container/richtext/richtext_state.rs b/crates/loro-internal/src/container/richtext/richtext_state.rs index fb8dad91b..db9f02f01 100644 --- a/crates/loro-internal/src/container/richtext/richtext_state.rs +++ b/crates/loro-internal/src/container/richtext/richtext_state.rs @@ -7,6 +7,7 @@ use loro_common::{Counter, IdFull, IdSpan, LoroError, LoroResult, LoroValue, ID} use query::{ByteQuery, ByteQueryT}; use rustc_hash::{FxHashMap, FxHashSet}; use serde::{ser::SerializeStruct, Serialize}; +use smallvec::SmallVec; use std::{ fmt::{Display, Formatter}, ops::{Bound, RangeBounds}, @@ -16,7 +17,6 @@ use std::{ str::Utf8Error, sync::Arc, }; -use smallvec::SmallVec; use tracing::instrument; use crate::{ diff --git a/crates/loro-internal/src/handler.rs b/crates/loro-internal/src/handler.rs index a86e9bb2d..001a36483 100644 --- a/crates/loro-internal/src/handler.rs +++ b/crates/loro-internal/src/handler.rs @@ -1854,12 +1854,9 @@ impl TextHandler { /// /// This method requires auto_commit to be enabled. pub fn splice(&self, pos: usize, len: usize, s: &str, pos_type: PosType) -> LoroResult { - let end = checked_range_end( - pos, - len, - self.len(pos_type), - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, self.len(pos_type), || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; let x = self.slice(pos, end, pos_type)?; self.delete(pos, len, pos_type)?; self.insert(pos, s, pos_type)?; @@ -1968,12 +1965,9 @@ impl TextHandler { } let text_len = self.len(pos_type); - let end = checked_range_end( - pos, - len, - text_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, text_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; self.validate_text_boundary(pos, pos_type)?; self.validate_text_boundary(end, pos_type)?; @@ -2236,12 +2230,9 @@ impl TextHandler { } let text_len = self.len(pos_type); - let end = checked_range_end( - pos, - len, - text_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - ) + let end = checked_range_end(pos, len, text_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + }) .inspect_err(|_| error!("pos={} len={} len={}", pos, len, text_len))?; self.validate_text_boundary(pos, pos_type)?; self.validate_text_boundary(end, pos_type)?; @@ -3245,12 +3236,9 @@ impl ListHandler { match &self.inner { MaybeDetached::Detached(l) => { let mut list = l.lock(); - let end = checked_range_end( - pos, - len, - list.value.len(), - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, list.value.len(), || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; list.value.drain(pos..end); Ok(()) } @@ -3264,12 +3252,9 @@ impl ListHandler { } let list_len = self.len(); - let end = checked_range_end( - pos, - len, - list_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, list_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; let inner = self.inner.try_attached_state()?; let ids: Vec<_> = inner.with_state(|state| { @@ -3919,12 +3904,9 @@ impl MovableListHandler { match &self.inner { MaybeDetached::Detached(d) => { let mut d = d.lock(); - let end = checked_range_end( - pos, - len, - d.value.len(), - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, d.value.len(), || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; d.value.drain(pos..end); Ok(()) } @@ -3939,12 +3921,9 @@ impl MovableListHandler { } let list_len = self.len(); - let end = checked_range_end( - pos, - len, - list_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, list_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; let (ids, new_poses) = self.with_state(|state| { let list = state.as_movable_list_state().unwrap(); diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index 0e5009d99..420f1267b 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -524,7 +524,26 @@ impl DocState { /// Panic when the diff cannot be merged with the previous diff. /// Caller should call [pre_txn] before calling this to avoid panic. fn record_diff(&mut self, diff: InternalDocDiff) { - if !self.event_recorder.recording_diff || diff.diff.is_empty() { + if !self.event_recorder.recording_diff { + return; + } + + if diff.diff.is_empty() { + let new_version = (*diff.new_version).to_owned(); + let Some(last_diff) = self.event_recorder.diffs.last_mut() else { + // A state-neutral import still advances the document version. Keep + // the next visible event's `from` frontier aligned with DocState. + self.event_recorder.diff_start_version = Some(new_version); + return; + }; + + if last_diff.origin != diff.origin || !last_diff.can_merge(&diff) { + panic!("should call pre_txn before record_diff"); + } + + // The empty transition belongs to the current event batch. It changes + // no container diff, but its version is still the batch's `to` frontier. + last_diff.new_version = Cow::Owned(new_version); return; } diff --git a/crates/loro-internal/src/state/map_state.rs b/crates/loro-internal/src/state/map_state.rs index 947eb6bcc..d82492a3a 100644 --- a/crates/loro-internal/src/state/map_state.rs +++ b/crates/loro-internal/src/state/map_state.rs @@ -29,9 +29,7 @@ fn translate_with_parent( value: LoroValue, ) -> LoroValue { match parent_id { - Some(parent) => { - loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value) - } + Some(parent) => loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value), None => value, } } diff --git a/crates/loro-internal/src/state/richtext_state.rs b/crates/loro-internal/src/state/richtext_state.rs index fac441bde..0060669fa 100644 --- a/crates/loro-internal/src/state/richtext_state.rs +++ b/crates/loro-internal/src/state/richtext_state.rs @@ -12,9 +12,7 @@ use crate::{ list::list_op, richtext::{ config::StyleConfigMap, - richtext_state::{ - DrainInfo, EntityRanges, IterRangeItem, PosType, RichtextStateChunk, - }, + richtext_state::{DrainInfo, EntityRanges, IterRangeItem, PosType, RichtextStateChunk}, AnchorType, RichtextState as InnerState, StyleKey, StyleOp, Styles, }, }, diff --git a/crates/loro-internal/src/state/tree_state.rs b/crates/loro-internal/src/state/tree_state.rs index e89964a2c..a4207c85f 100644 --- a/crates/loro-internal/src/state/tree_state.rs +++ b/crates/loro-internal/src/state/tree_state.rs @@ -281,25 +281,6 @@ impl NodeChildren { } } - fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { - match self { - NodeChildren::Vec(v) => { - if v.len() >= Self::MAX_SIZE_FOR_ARRAY { - self.upgrade(); - return self.push_child_in_order(pos, id); - } - - if let Some(last) = v.last() { - assert!(last.0 < pos); - } - v.push((pos, id)); - } - NodeChildren::BTree(v) => { - v.push_child_in_order(pos, id); - } - } - } - fn len(&self) -> usize { match self { NodeChildren::Vec(v) => v.len(), @@ -366,14 +347,6 @@ mod btree { self.id_to_leaf_index.insert(id, c.leaf); } - pub(super) fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { - let c = self.tree.push(Elem { - pos: Arc::new(pos.clone()), - id, - }); - self.id_to_leaf_index.insert(id, c.leaf); - } - pub(super) fn delete_child(&mut self, id: &TreeID) { if let Some(leaf) = self.id_to_leaf_index.remove(id) { self.tree.remove_leaf(Cursor { leaf, offset: 0 }); @@ -710,19 +683,28 @@ impl TreeState { Ok(()) } - fn _init_push_tree_node_in_order( + fn _init_tree_node_from_snapshot( &mut self, target: TreeID, parent: TreeParentId, last_move_op: IdFull, position: Option, ) -> Result<(), LoroError> { - debug_assert!(!self.trees.contains_key(&target)); + if self.trees.contains_key(&target) { + return Err(LoroError::internal("duplicate tree node id in snapshot")); + } let entry = self.children.entry(parent).or_default(); let node_position = NodePosition::new(position.clone().unwrap_or_default(), last_move_op.idlp()); - debug_assert!(!entry.has_child(&node_position)); - entry.push_child_in_order(node_position, target); + if entry.has_child(&node_position) { + return Err(LoroError::internal( + "duplicate tree node position in snapshot", + )); + } + // Snapshot bytes are external input. Canonical encoders group siblings + // in order, but interoperable encoders may choose another node traversal. + // Insert by NodePosition instead of asserting that input order is sorted. + entry.insert_child(node_position, target); self.trees.insert( target, TreeStateNode { @@ -1726,7 +1708,7 @@ mod snapshot { for (node_id, node) in node_ids.iter().zip(encoded.nodes) { // PERF: we don't need to mov the deleted node, instead we can cache them // If the parent is TreeParentId::Deleted, then all the nodes afterwards are deleted - tree._init_push_tree_node_in_order( + tree._init_tree_node_from_snapshot( *node_id, match node.parent_idx_plus_two { 0 => TreeParentId::Root, @@ -1827,5 +1809,60 @@ mod snapshot { assert!(TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &bytes), ctx).is_err()); } + + #[test] + fn tree_fast_snapshot_accepts_noncanonical_sibling_order() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Tree); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + let low = FractionalIndex::default(); + let high = FractionalIndex::from_bytes(vec![0x81]); + let positions = PositionArena::from_positions([high.as_bytes(), low.as_bytes()]); + let encoded = EncodedTree { + node_ids: vec![ + EncodedTreeNodeId { + peer_idx: 0, + counter: 0, + }, + EncodedTreeNodeId { + peer_idx: 0, + counter: 1, + }, + ], + nodes: vec![ + EncodedTreeNode { + parent_idx_plus_two: 0, + last_set_peer_idx: 0, + last_set_counter: 0, + last_set_lamport_sub_counter: 0, + fractional_index_idx: 0, + }, + EncodedTreeNode { + parent_idx_plus_two: 0, + last_set_peer_idx: 0, + last_set_counter: 1, + last_set_lamport_sub_counter: 0, + fractional_index_idx: 1, + }, + ], + fractional_indexes: positions.encode().into(), + reserved_has_effect_bool_rle: vec![].into(), + }; + let mut bytes = Vec::new(); + leb128::write::unsigned(&mut bytes, 1).unwrap(); + bytes.extend_from_slice(&1_u64.to_le_bytes()); + bytes.extend_from_slice(&serde_columnar::to_vec(&encoded).unwrap()); + + let tree = + TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &bytes), ctx).unwrap(); + let roots = tree.tree_nodes(); + assert_eq!(roots.len(), 2); + assert_eq!(roots[0].id.counter, 1); + assert_eq!(roots[1].id.counter, 0); + } } } diff --git a/crates/loro-internal/src/txn.rs b/crates/loro-internal/src/txn.rs index 817b53cb1..aa600cc0f 100644 --- a/crates/loro-internal/src/txn.rs +++ b/crates/loro-internal/src/txn.rs @@ -10,7 +10,7 @@ use loro_common::{ContainerType, IdLp, IdSpan, LoroResult}; use loro_delta::{array_vec::ArrayVec, DeltaRopeBuilder}; use rle::{HasLength, Mergable, RleVec, Sliceable}; use rustc_hash::FxHashMap; -use smallvec::{smallvec, SmallVec}; +use smallvec::SmallVec; use crate::{ change::{Change, Lamport, Timestamp}, @@ -569,7 +569,12 @@ impl Transaction { let expected = self .doc .upgrade() - .map(|d| d.state.lock().peer.load(std::sync::atomic::Ordering::Relaxed)) + .map(|d| { + d.state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed) + }) .unwrap_or(found); return Err(LoroError::UnmatchedContext { expected, found }); } @@ -770,43 +775,35 @@ fn change_to_diff( }; let mut op_index = 0; - let mut hint_iter = hints.into_iter(); - let mut current_hint = hint_iter.next(); - - while op_index < container_ops.len() { - let Some(hint) = current_hint.take() else { - unreachable!("Missing hint for op"); - }; - - // Collect ops that belong to this hint - let mut ops_for_hint: SmallVec<[Op; 1]> = smallvec![container_ops[op_index].clone()]; - let mut total_len = container_ops[op_index].atom_len(); - - // If hint spans multiple ops, collect them - while total_len < hint.rle_len() { - op_index += 1; - let next_op_len = container_ops[op_index].atom_len(); - let op = if next_op_len + total_len > hint.rle_len() { - let new_len = hint.rle_len() - total_len; - let left = container_ops[op_index].slice(0, new_len); - let right = container_ops[op_index].slice(new_len, next_op_len); - container_ops[op_index] = right; - op_index -= 1; - left + let mut op_offset = 0; + + for hint in hints { + // Ops and event hints use independent RLE merging rules. Walk the op + // stream by atom offset so a single merged op can be split across + // multiple hints (notably adjacent UTF-16 text deletions). + let mut remaining = hint.rle_len(); + let mut ops_for_hint: SmallVec<[Op; 1]> = SmallVec::new(); + while remaining > 0 { + let op = container_ops + .get(op_index) + .unwrap_or_else(|| unreachable!("Missing op for hint")); + let op_len = op.atom_len(); + let available = op_len - op_offset; + let take = available.min(remaining); + let piece = if op_offset == 0 && take == op_len { + op.clone() } else { - container_ops[op_index].clone() + op.slice(op_offset, op_offset + take) }; - - total_len += op.atom_len(); - ops_for_hint.push(op); + ops_for_hint.push(piece); + remaining -= take; + op_offset += take; + if op_offset == op_len { + op_index += 1; + op_offset = 0; + } } - op_index += 1; - assert_eq!(total_len, hint.rle_len(), "Op/hint length mismatch"); - - // Move to next hint - current_hint = hint_iter.next(); - // Generate diff based on hint type match hint { EventHint::Mark { start, end, style } => { @@ -981,6 +978,9 @@ fn change_to_diff( .map(|x| x.content_len() as Lamport) .sum::(); } + + assert_eq!(op_offset, 0, "Missing hint for partial op"); + assert_eq!(op_index, container_ops.len(), "Missing hint for op"); } ans @@ -1015,4 +1015,19 @@ mod tests { )); assert!(!doc.app_state().lock().is_in_txn()); } + + #[cfg(feature = "wasm")] + #[test] + fn txn_events_split_merged_utf16_delete_ops() { + let doc = LoroDoc::new(); + let _subscription = doc.subscribe_root(Arc::new(|_| {})); + let text = doc.get_text("text"); + + text.insert_utf16(0, "a𐐷").unwrap(); + text.delete_utf16(1, 2).unwrap(); + text.delete_utf16(0, 1).unwrap(); + doc.commit_then_renew(); + + assert_eq!(text.to_string(), ""); + } } diff --git a/crates/loro-internal/tests/mergeable_container/convergence.rs b/crates/loro-internal/tests/mergeable_container/convergence.rs index d2ad233f7..db9a55512 100644 --- a/crates/loro-internal/tests/mergeable_container/convergence.rs +++ b/crates/loro-internal/tests/mergeable_container/convergence.rs @@ -279,10 +279,7 @@ fn detached_map_ensure_mergeable_counter_rejects_misuse() { .ensure_mergeable_counter("revision") .expect_err("detached ensure_mergeable_* must error"); assert!( - matches!( - err, - loro_common::LoroError::MisuseDetachedContainer { .. } - ), + matches!(err, loro_common::LoroError::MisuseDetachedContainer { .. }), "detached error must be MisuseDetachedContainer; got {err:?}" ); } diff --git a/crates/loro-internal/tests/mergeable_container/delete.rs b/crates/loro-internal/tests/mergeable_container/delete.rs index a032e7d58..0ca6f2a2a 100644 --- a/crates/loro-internal/tests/mergeable_container/delete.rs +++ b/crates/loro-internal/tests/mergeable_container/delete.rs @@ -261,7 +261,10 @@ fn three_peer_delete_then_recreate_converges() { sync(&a, &c); // C recreates "revision" as a text with a post-delete IdLp. - let c_text = c.get_map("state").ensure_mergeable_text("revision").unwrap(); + let c_text = c + .get_map("state") + .ensure_mergeable_text("revision") + .unwrap(); c_text.insert(0, "after-delete", PosType::Unicode).unwrap(); c.commit_then_renew(); diff --git a/crates/loro-internal/tests/mergeable_container/snapshot.rs b/crates/loro-internal/tests/mergeable_container/snapshot.rs index f45a52288..811cb5738 100644 --- a/crates/loro-internal/tests/mergeable_container/snapshot.rs +++ b/crates/loro-internal/tests/mergeable_container/snapshot.rs @@ -291,9 +291,7 @@ fn shallow_snapshot_preserves_losing_kind_state() { // The Map marker now wins LWW on A and is the visible kind; the Text child becomes the loser // whose state lives only at its deterministic cid. for i in 0..6 { - a.get_map("state") - .insert(&format!("noise_{i}"), i) - .unwrap(); + a.get_map("state").insert(&format!("noise_{i}"), i).unwrap(); a.commit_then_renew(); } let a_map = a.get_map("state").ensure_mergeable_map("k").unwrap(); diff --git a/crates/loro-internal/tests/test.rs b/crates/loro-internal/tests/test.rs index 8cd8809b0..3f45bcd73 100644 --- a/crates/loro-internal/tests/test.rs +++ b/crates/loro-internal/tests/test.rs @@ -125,6 +125,38 @@ fn event_from_checkout() { assert!(ran_cloned.load(std::sync::atomic::Ordering::Relaxed)); } +#[test] +fn local_event_starts_after_net_empty_import() { + let source = LoroDoc::new_auto_commit(); + source.set_peer_id(2).unwrap(); + let source_list = source.get_list("source"); + source_list.insert(0, false).unwrap(); + source_list.delete(0, 1).unwrap(); + source.commit_then_renew(); + let imported_frontiers = source.oplog_frontiers(); + + let target = LoroDoc::new_auto_commit(); + target.set_peer_id(1).unwrap(); + let saw_local = Arc::new(AtomicBool::new(false)); + let saw_local_in_callback = saw_local.clone(); + let expected_from = imported_frontiers.clone(); + let _sub = target.subscribe_root(Arc::new(move |event| { + if event.event_meta.by.is_local() { + assert_eq!(event.event_meta.from, expected_from); + saw_local_in_callback.store(true, std::sync::atomic::Ordering::Relaxed); + } + })); + + target + .import(&source.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!(target.oplog_frontiers(), imported_frontiers); + target.get_list("visible").insert(0, true).unwrap(); + target.commit_then_renew(); + + assert!(saw_local.load(std::sync::atomic::Ordering::Relaxed)); +} + #[test] fn handler_in_event() { let doc = LoroDoc::new_auto_commit(); diff --git a/crates/loro/tests/mergeable_public_api.rs b/crates/loro/tests/mergeable_public_api.rs index 12d87ffbc..f98a4f9e0 100644 --- a/crates/loro/tests/mergeable_public_api.rs +++ b/crates/loro/tests/mergeable_public_api.rs @@ -215,8 +215,14 @@ fn loro_map_ensure_mergeable_tree_through_public_api() { let a = doc(1); let b = doc(2); - let a_tree = a.get_map("state").ensure_mergeable_tree("hierarchy").unwrap(); - let b_tree = b.get_map("state").ensure_mergeable_tree("hierarchy").unwrap(); + let a_tree = a + .get_map("state") + .ensure_mergeable_tree("hierarchy") + .unwrap(); + let b_tree = b + .get_map("state") + .ensure_mergeable_tree("hierarchy") + .unwrap(); assert_eq!( a_tree.id(), b_tree.id(), diff --git a/loro-js/fuzz/README.md b/loro-js/fuzz/README.md index 3d908e6b7..f957c332e 100644 --- a/loro-js/fuzz/README.md +++ b/loro-js/fuzz/README.md @@ -59,7 +59,10 @@ not canonical. Equivalent event representations are normalized: adjacent delta fragments are coalesced, insert/delete operations at one cursor are put in a stable order, -and transaction-local tree create/delete pairs are treated as a net no-op. +and transaction-local tree create/delete pairs are treated as a net no-op. The +normalizer only reads event payloads; it never queries live document state from +inside a subscription callback, because the WASM wrapper and pure TypeScript +runtime can reach the callback at different points within one public method. Snapshot imports into a non-empty document still have known event differences, so the stable profile compares their resulting state but not their event list. diff --git a/loro-js/fuzz/corpus/checkout-pending-tree.json b/loro-js/fuzz/corpus/checkout-pending-tree.json new file mode 100644 index 000000000..42522e35b --- /dev/null +++ b/loro-js/fuzz/corpus/checkout-pending-tree.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "save", "peer": 0, "checkpoint": 0 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": 1 }, + { "kind": "checkout", "peer": 0, "checkpoint": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/checkout-tree-metadata-revival.json b/loro-js/fuzz/corpus/checkout-tree-metadata-revival.json new file mode 100644 index 000000000..211823528 --- /dev/null +++ b/loro-js/fuzz/corpus/checkout-tree-metadata-revival.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 129, "parent": null, "value": {} }, + { "kind": "save", "peer": 51, "checkpoint": 0 }, + { "kind": "tree-delete", "peer": 33, "node": 0 }, + { "kind": "checkout", "peer": 21, "checkpoint": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/checkout-tree-old-indices.json b/loro-js/fuzz/corpus/checkout-tree-old-indices.json new file mode 100644 index 000000000..0e9ae873c --- /dev/null +++ b/loro-js/fuzz/corpus/checkout-tree-old-indices.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "save", "peer": 0, "checkpoint": 3 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": {} }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": {} }, + { "kind": "checkout", "peer": 0, "checkpoint": 3 } + ] +} diff --git a/loro-js/fuzz/corpus/concurrent-tree-reindex.json b/loro-js/fuzz/corpus/concurrent-tree-reindex.json new file mode 100644 index 000000000..93279720c --- /dev/null +++ b/loro-js/fuzz/corpus/concurrent-tree-reindex.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 15, "parent": null, "value": [] }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "enqueue", "source": 0, "target": 0, "slot": 4, "mode": "update" }, + { "kind": "deliver", "slot": 4, "copies": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/deleted-tree-position.json b/loro-js/fuzz/corpus/deleted-tree-position.json new file mode 100644 index 000000000..d0a18ea95 --- /dev/null +++ b/loro-js/fuzz/corpus/deleted-tree-position.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "tree-create", + "peer": 0, + "parent": null, + "value": {} + }, + { + "kind": "tree-create", + "peer": 0, + "parent": null, + "value": false + }, + { + "kind": "tree-delete", + "peer": 0, + "node": 1 + }, + { + "kind": "commit", + "peer": 0, + "message": 0 + } + ] +} diff --git a/loro-js/fuzz/corpus/empty-import-event-frontier.json b/loro-js/fuzz/corpus/empty-import-event-frontier.json new file mode 100644 index 000000000..5da46ab84 --- /dev/null +++ b/loro-js/fuzz/corpus/empty-import-event-frontier.json @@ -0,0 +1,43 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "list-insert", + "peer": 57, + "index": 0, + "value": false + }, + { + "kind": "list-delete", + "peer": 21, + "index": 0, + "length": 0 + }, + { + "kind": "enqueue", + "source": 111, + "target": 0, + "slot": 4, + "mode": "update" + }, + { + "kind": "deliver", + "slot": 4, + "copies": 0 + }, + { + "kind": "movable-insert", + "peer": 0, + "index": 0, + "value": {} + }, + { + "kind": "enqueue", + "source": 0, + "target": 0, + "slot": 0, + "mode": "update" + } + ] +} diff --git a/loro-js/fuzz/corpus/import-tree-replay-noop.json b/loro-js/fuzz/corpus/import-tree-replay-noop.json new file mode 100644 index 000000000..e8c772a03 --- /dev/null +++ b/loro-js/fuzz/corpus/import-tree-replay-noop.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "map-set", + "peer": 1, + "key": "a", + "value": false + }, + { + "kind": "enqueue", + "source": 1, + "target": 0, + "slot": 0, + "mode": "update" + }, + { + "kind": "tree-create", + "peer": 0, + "parent": null, + "value": "" + }, + { + "kind": "deliver", + "slot": 0, + "copies": 0 + } + ] +} diff --git a/loro-js/fuzz/corpus/local-map-delete-restore-event.json b/loro-js/fuzz/corpus/local-map-delete-restore-event.json new file mode 100644 index 000000000..0a8500f60 --- /dev/null +++ b/loro-js/fuzz/corpus/local-map-delete-restore-event.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "map-set", + "peer": 69, + "key": "title", + "value": {} + }, + { + "kind": "attach", + "peer": 73 + }, + { + "kind": "map-delete", + "peer": 3, + "key": "a" + }, + { + "kind": "map-set", + "peer": 13, + "key": "title", + "value": {} + }, + { + "kind": "attach", + "peer": 217 + } + ] +} diff --git a/loro-js/fuzz/corpus/local-tree-nested-create.json b/loro-js/fuzz/corpus/local-tree-nested-create.json new file mode 100644 index 000000000..ead52b8df --- /dev/null +++ b/loro-js/fuzz/corpus/local-tree-nested-create.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 45, "parent": null, "value": [] }, + { "kind": "tree-create", "peer": 105, "parent": null, "value": null }, + { "kind": "tree-move", "peer": 3, "node": 0, "parent": 0 }, + { "kind": "commit", "peer": 51, "message": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/local-tree-parent-reindex.json b/loro-js/fuzz/corpus/local-tree-parent-reindex.json new file mode 100644 index 000000000..e42b41c88 --- /dev/null +++ b/loro-js/fuzz/corpus/local-tree-parent-reindex.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "commit", "peer": 0, "message": 0 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "tree-move", "peer": 0, "node": 0, "parent": 0 }, + { "kind": "enqueue", "source": 0, "target": 0, "slot": 0, "mode": "update" } + ] +} diff --git a/loro-js/fuzz/corpus/retain-only-event.json b/loro-js/fuzz/corpus/retain-only-event.json new file mode 100644 index 000000000..abc1bd3a1 --- /dev/null +++ b/loro-js/fuzz/corpus/retain-only-event.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "list-insert", "peer": 217, "index": 0, "value": 0 }, + { "kind": "attach", "peer": 45 }, + { "kind": "list-insert", "peer": 7, "index": 93, "value": {} }, + { "kind": "list-delete", "peer": 3, "index": 15, "length": 0 }, + { "kind": "attach", "peer": 189 } + ] +} diff --git a/loro-js/fuzz/corpus/snapshot-tree-sibling-order.json b/loro-js/fuzz/corpus/snapshot-tree-sibling-order.json new file mode 100644 index 000000000..f9c907803 --- /dev/null +++ b/loro-js/fuzz/corpus/snapshot-tree-sibling-order.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 1, "parent": null, "value": [] }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "deliver", "slot": 0, "copies": 0 }, + { "kind": "roundtrip", "peer": 1, "mode": "snapshot" } + ] +} diff --git a/loro-js/fuzz/corpus/trailing-retain-event.json b/loro-js/fuzz/corpus/trailing-retain-event.json new file mode 100644 index 000000000..ca2f552d5 --- /dev/null +++ b/loro-js/fuzz/corpus/trailing-retain-event.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "movable-insert", + "peer": 33, + "index": 0, + "value": [] + }, + { + "kind": "attach", + "peer": 189 + }, + { + "kind": "movable-insert", + "peer": 189, + "index": 91, + "value": [] + }, + { + "kind": "movable-move", + "peer": 33, + "from": 153, + "to": 0 + }, + { + "kind": "attach", + "peer": 73 + }, + { + "kind": "text-delete", + "peer": 0, + "index": 0, + "length": 0 + } + ] +} diff --git a/loro-js/fuzz/corpus/transient-tree-reindex.json b/loro-js/fuzz/corpus/transient-tree-reindex.json new file mode 100644 index 000000000..62767e937 --- /dev/null +++ b/loro-js/fuzz/corpus/transient-tree-reindex.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "peerCount": 3, + "commands": [ + { + "kind": "tree-create", + "peer": 1, + "parent": null, + "value": false + }, + { + "kind": "tree-create", + "peer": 1, + "parent": null, + "value": false + }, + { + "kind": "tree-delete", + "peer": 1, + "node": 0 + }, + { + "kind": "enqueue", + "source": 1, + "target": 0, + "slot": 0, + "mode": "update" + } + ] +} diff --git a/loro-js/fuzz/corpus/transient-tree-subtree.json b/loro-js/fuzz/corpus/transient-tree-subtree.json new file mode 100644 index 000000000..e0cc69907 --- /dev/null +++ b/loro-js/fuzz/corpus/transient-tree-subtree.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "tree-create", + "peer": 1, + "parent": null, + "value": [] + }, + { + "kind": "tree-create", + "peer": 1, + "parent": 0, + "value": null + }, + { + "kind": "tree-delete", + "peer": 1, + "node": 0 + }, + { + "kind": "enqueue", + "source": 1, + "target": 0, + "slot": 0, + "mode": "update" + } + ] +} diff --git a/loro-js/fuzz/corpus/transient-tree.json b/loro-js/fuzz/corpus/transient-tree.json new file mode 100644 index 000000000..a2bae246a --- /dev/null +++ b/loro-js/fuzz/corpus/transient-tree.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 1, "parent": null, "value": [] }, + { "kind": "tree-delete", "peer": 1, "node": 0 }, + { "kind": "commit", "peer": 1, "message": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/utf16-delete-hint-split.json b/loro-js/fuzz/corpus/utf16-delete-hint-split.json new file mode 100644 index 000000000..251ffb255 --- /dev/null +++ b/loro-js/fuzz/corpus/utf16-delete-hint-split.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "text-insert", + "peer": 1, + "index": 0, + "text": "a𐐷" + }, + { + "kind": "text-delete", + "peer": 1, + "index": 1, + "length": 0 + }, + { + "kind": "text-delete", + "peer": 1, + "index": 0, + "length": 0 + }, + { + "kind": "commit", + "peer": 1, + "message": 0 + } + ] +} diff --git a/loro-js/fuzz/run-interop-fuzz.mjs b/loro-js/fuzz/run-interop-fuzz.mjs index 98b11ab39..0f2c9f888 100644 --- a/loro-js/fuzz/run-interop-fuzz.mjs +++ b/loro-js/fuzz/run-interop-fuzz.mjs @@ -1,5 +1,14 @@ import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + writeFileSync, + writeSync, +} from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -40,6 +49,7 @@ const wasmEngine = : importedWasmEngine; const corpus = readCorpus(options.replay, options.strict); const malformedCases = runMalformedImportChecks(wasmEngine, jsEngine); +let randomRuns = 0; for (const { name, scenario } of corpus) { runDifferentialScenario(wasmEngine, jsEngine, scenario, { @@ -51,19 +61,36 @@ for (const { name, scenario } of corpus) { } if (!options.corpusOnly) { + mkdirSync(artifactsRoot, { recursive: true }); + const journalPath = path.join(artifactsRoot, "active-run.txt"); + const journal = openSync(journalPath, "w"); + let runIndex = 0; const arbitrary = scenarioArbitrary(options.maxCommands, { strict: options.strict }); const property = fc.property(arbitrary, (scenario) => { + // A Rust panic can poison a lock before fast-check returns its seed/path. + // Persist the deterministic raw-case position first so a killed or wedged + // process can be replayed with `seed` plus `path=`. + writeSync(journal, `${options.seed} ${runIndex}\n`, 0, "utf8"); + runIndex += 1; runDifferentialScenario(wasmEngine, jsEngine, scenario, { strict: options.strict, }); }); - const check = fc.check(property, { - numRuns: options.runs, - seed: options.seed, - path: options.path, - endOnFailure: true, - interruptAfterTimeLimit: options.timeLimit, - }); + process.stdout.write( + `loro interop fuzz started: seed=${options.seed} maxCommands=${options.maxCommands}\n`, + ); + let check; + try { + check = fc.check(property, { + numRuns: options.runs, + seed: options.seed, + path: options.path, + interruptAfterTimeLimit: options.timeLimit, + }); + } finally { + closeSync(journal); + } + randomRuns = check.numRuns; if (check.failed) { mkdirSync(artifactsRoot, { recursive: true }); const scenario = check.counterexample?.[0]; @@ -83,7 +110,7 @@ if (!options.corpusOnly) { } process.stdout.write( - `loro interop fuzz passed: profile=${options.strict ? "strict" : "stable"} corpus=${corpus.length} malformed=${malformedCases} random=${options.corpusOnly ? 0 : options.runs}\n`, + `loro interop fuzz passed: profile=${options.strict ? "strict" : "stable"} corpus=${corpus.length} malformed=${malformedCases} random=${randomRuns}\n`, ); function parseArgs(args) { @@ -106,7 +133,7 @@ function parseArgs(args) { replay, runs: Number.isSafeInteger(envRuns) ? envRuns : ci ? 500 : 100, maxCommands: Number.isSafeInteger(envCommands) ? envCommands : ci ? 60 : 40, - seed: Number.isSafeInteger(envSeed) ? envSeed : undefined, + seed: Number.isSafeInteger(envSeed) ? envSeed : (Date.now() ^ process.pid) | 0, path: process.env.LORO_INTEROP_FUZZ_PATH, timeLimit: Number.isSafeInteger(envTime) ? envTime : ci ? 60_000 : undefined, }; diff --git a/loro-js/fuzz/runner.mjs b/loro-js/fuzz/runner.mjs index 722086357..fd4b8e97a 100644 --- a/loro-js/fuzz/runner.mjs +++ b/loro-js/fuzz/runner.mjs @@ -48,6 +48,26 @@ function canonicalVersion(version) { return canonicalize(json); } +function releaseResource(resource) { + resource?.free?.(); +} + +function releaseResources(resources) { + for (const resource of resources) releaseResource(resource); +} + +function usingResource(resource, action) { + try { + return action(resource); + } finally { + releaseResource(resource); + } +} + +function canonicalVersionAndRelease(version) { + return usingResource(version, canonicalVersion); +} + function canonicalFrontiers(frontiers) { return frontiers .map(({ peer, counter }) => `${counter}@${peer}`) @@ -68,7 +88,7 @@ function normalizeDeepWithId(value, key) { ); } -function normalizeEventBatch(batch, doc) { +function normalizeEventBatch(batch) { const normalized = canonicalize(batch); // `origin` is binding metadata rather than a CRDT diff. The pure TS runtime // does not currently expose every synthetic Rust origin (for example @@ -82,16 +102,20 @@ function normalizeEventBatch(batch, doc) { } } if (Array.isArray(normalized.events)) { + const deletedTreeNodes = collectDeletedTreeNodes(normalized.events); for (const event of normalized.events) { if (Array.isArray(event.diff?.diff)) { - event.diff.diff = canonicalizeDeltaOrder(coalesceDelta(event.diff.diff)); + event.diff.diff = trimPlainTrailingRetains( + canonicalizeDeltaOrder(coalesceDelta(event.diff.diff)), + ); } if (event.diff?.type === "tree" && Array.isArray(event.diff.diff)) { event.diff.diff = removeTransientTreeActions(event.diff.diff); } } normalized.events = normalized.events.filter( - (event) => eventHasEffect(event) && !targetsDeletedTreeNode(event, doc), + (event) => + eventHasEffect(event) && !targetsTreeNodeDeletedInBatch(event, deletedTreeNodes), ); normalized.events.sort((left, right) => JSON.stringify([left.target, left.path]).localeCompare( @@ -102,6 +126,21 @@ function normalizeEventBatch(batch, doc) { return canonicalize(normalized); } +function collectDeletedTreeNodes(events) { + const deleted = new Set(); + for (const event of events) { + if (event.diff?.type !== "tree" || !Array.isArray(event.diff.diff)) continue; + const root = event.path?.[0]; + if (typeof root !== "string") continue; + for (const action of event.diff.diff) { + if (action.action === "delete" && typeof action.target === "string") { + deleted.add(JSON.stringify([root, action.target])); + } + } + } + return deleted; +} + function canonicalizeDeltaOrder(delta) { const output = []; let group = []; @@ -127,34 +166,230 @@ function canonicalizeDeltaOrder(delta) { } function removeTransientTreeActions(actions) { + const actionsByTarget = new Map(); + for (const action of actions) { + let targetActions = actionsByTarget.get(action.target); + if (targetActions === undefined) { + targetActions = []; + actionsByTarget.set(action.target, targetActions); + } + targetActions.push(action); + } + const finalParents = new Map(); + const finalDeleted = new Set(); + for (const action of actions) { + if (action.action === "delete") { + finalDeleted.add(action.target); + } else { + finalDeleted.delete(action.target); + finalParents.set(action.target, normalizeTreeParent(action.parent)); + } + } + const endsDeleted = (target, visiting = new Set()) => { + if (finalDeleted.has(target)) return true; + const parent = finalParents.get(target); + if (parent === undefined || parent === null || visiting.has(parent)) return false; + visiting.add(parent); + return endsDeleted(parent, visiting); + }; const transientTargets = new Set(); + for (const [target, targetActions] of actionsByTarget) { + if (targetActions[0]?.action === "create" && endsDeleted(target)) { + transientTargets.add(target); + } + } + + const positions = new Map(); + const countBefore = (parent, index, target) => { + let count = 0; + for (const [candidate, position] of positions) { + if ( + candidate !== target && + transientTargets.has(candidate) && + position.parent === parent && + position.index < index + ) { + count += 1; + } + } + return count; + }; + const removeAt = (parent, index, target) => { + positions.delete(target); + for (const position of positions.values()) { + if (position.parent === parent && position.index > index) position.index -= 1; + } + }; + const insertAt = (parent, index, target) => { + for (const [candidate, position] of positions) { + if (candidate !== target && position.parent === parent && position.index >= index) { + position.index += 1; + } + } + positions.set(target, { parent, index }); + }; + + const output = []; for (const action of actions) { - if (action.action !== "delete") continue; - if ( - actions.some( - (candidate) => - candidate.target === action.target && candidate.action === "create", - ) - ) { - transientTargets.add(action.target); + const transient = transientTargets.has(action.target); + const normalized = structuredClone(action); + if (action.action === "create") { + const parent = normalizeTreeParent(action.parent); + if (!transient) { + normalized.index -= countBefore(parent, action.index, action.target); + } + insertAt(parent, action.index, action.target); + } else if (action.action === "delete") { + const oldParent = normalizeTreeParent(action.oldParent); + if (!transient) { + normalized.oldIndex -= countBefore(oldParent, action.oldIndex, action.target); + } + const position = positions.get(action.target) ?? { + parent: oldParent, + index: action.oldIndex, + }; + removeAt(position.parent, position.index, action.target); + } else if (action.action === "move") { + const oldParent = normalizeTreeParent(action.oldParent); + const parent = normalizeTreeParent(action.parent); + if (!transient) { + normalized.oldIndex -= countBefore(oldParent, action.oldIndex, action.target); + } + const previous = positions.get(action.target) ?? { + parent: oldParent, + index: action.oldIndex, + }; + removeAt(previous.parent, previous.index, action.target); + if (!transient) { + normalized.index -= countBefore(parent, action.index, action.target); + } + insertAt(parent, action.index, action.target); } + if (!transient) output.push(normalized); } - return actions.filter((action) => !transientTargets.has(action.target)); + const hasComposedTarget = output.some((action, index) => + output.slice(index + 1).some((candidate) => candidate.target === action.target), + ); + return coalesceTreeActions( + output, + hasComposedTarget ? positions : undefined, + transientTargets, + ); } -function targetsDeletedTreeNode(event, doc) { +function coalesceTreeActions(actions, finalPositions, transientTargets) { + const actionsByTarget = new Map(); + for (const action of actions) { + let targetActions = actionsByTarget.get(action.target); + if (targetActions === undefined) { + targetActions = []; + actionsByTarget.set(action.target, targetActions); + } + targetActions.push(action); + } + const departures = []; + if (finalPositions === undefined) { + for (const [target, targetActions] of actionsByTarget) { + const first = targetActions[0]; + if (first.action === "create") continue; + const last = targetActions.at(-1); + const oldParent = normalizeTreeParent(first.oldParent); + const parent = + last.action === "delete" ? undefined : normalizeTreeParent(last.parent); + if (parent !== oldParent) { + departures.push({ target, parent: oldParent, index: first.oldIndex }); + } + } + } + const finalIndex = (target, parent, fallback) => { + const position = finalPositions?.get(target); + if (position === undefined) { + const removedBefore = departures.filter( + (departure) => + departure.target !== target && + departure.parent === parent && + departure.index < fallback, + ).length; + return fallback - removedBefore; + } + let removedBefore = 0; + for (const [candidate, candidatePosition] of finalPositions) { + if ( + transientTargets.has(candidate) && + candidatePosition.parent === position.parent && + candidatePosition.index < position.index + ) { + removedBefore += 1; + } + } + return position.index - removedBefore; + }; + const output = []; + for (const [target, targetActions] of actionsByTarget) { + const first = targetActions[0]; + const last = targetActions.at(-1); + const existedBefore = first.action !== "create"; + const existsAfter = last.action !== "delete"; + if (!existedBefore && !existsAfter) continue; + if (!existedBefore) { + output.push({ + target, + action: "create", + parent: last.parent, + index: finalIndex(target, normalizeTreeParent(last.parent), last.index), + fractional_index: last.fractional_index, + }); + continue; + } + const oldParent = normalizeTreeParent(first.oldParent); + const oldIndex = first.oldIndex; + if (!existsAfter) { + output.push({ target, action: "delete", oldParent, oldIndex }); + continue; + } + const parent = normalizeTreeParent(last.parent); + const index = finalIndex(target, parent, last.index); + if (parent === oldParent && index === oldIndex) continue; + output.push({ + target, + action: "move", + parent, + index, + fractional_index: last.fractional_index, + oldParent, + oldIndex, + }); + } + return output.sort((left, right) => left.target.localeCompare(right.target)); +} + +function normalizeTreeParent(parent) { + return parent ?? null; +} + +function targetsTreeNodeDeletedInBatch(event, deletedTreeNodes) { const [root, node] = event.path ?? []; return ( typeof root === "string" && typeof node === "string" && - node.includes("@") && - doc.getTree(root).isNodeDeleted(node) + deletedTreeNodes.has(JSON.stringify([root, node])) ); } function eventHasEffect(event) { const diff = event.diff; - if (Array.isArray(diff?.diff)) return diff.diff.length > 0; + if (Array.isArray(diff?.diff)) { + if (diff.type === "list" || diff.type === "text") { + return diff.diff.some( + (operation) => + operation.insert !== undefined || + operation.delete !== undefined || + (operation.attributes !== undefined && + Object.keys(operation.attributes).length > 0), + ); + } + return diff.diff.length > 0; + } if (diff?.type === "map") return Object.keys(diff.updated ?? {}).length > 0; if (diff?.type === "counter") return diff.increment !== 0; return true; @@ -201,39 +436,60 @@ function coalesceDelta(delta) { return output; } +function trimPlainTrailingRetains(delta) { + while (delta.length > 0) { + const operation = delta.at(-1); + if ( + typeof operation?.retain !== "number" || + (operation.attributes !== undefined && Object.keys(operation.attributes).length > 0) + ) { + break; + } + delta.pop(); + } + return delta; +} + export function observeDoc(doc) { const pendingLength = doc.getPendingTxnLength(); const tree = doc.getTree("tree"); - const treeNodes = tree - .getNodes({ withDeleted: true }) - .map((node) => node.toJSON()) - .sort((left, right) => String(left.id).localeCompare(String(right.id))); - const observation = { - json: doc.toJSON(), - deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), - textDelta: doc.getText("text").toDelta(), - treeNodes, - version: canonicalVersion(doc.version()), - frontiers: canonicalFrontiers(doc.frontiers()), - detached: doc.isDetached(), - shallow: doc.isShallow(), - pendingLength, - }; - if (pendingLength === 0) { - observation.oplogVersion = canonicalVersion(doc.oplogVersion()); - observation.oplogFrontiers = canonicalFrontiers(doc.oplogFrontiers()); - observation.opCount = doc.opCount(); - observation.changeCount = doc.changeCount(); + const nodes = tree.getNodes({ withDeleted: true }); + const text = doc.getText("text"); + try { + const treeNodes = nodes + .map((node) => node.toJSON()) + .sort((left, right) => String(left.id).localeCompare(String(right.id))); + const observation = { + json: doc.toJSON(), + deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), + textDelta: text.toDelta(), + treeNodes, + version: canonicalVersionAndRelease(doc.version()), + frontiers: canonicalFrontiers(doc.frontiers()), + detached: doc.isDetached(), + shallow: doc.isShallow(), + pendingLength, + }; + if (pendingLength === 0) { + observation.oplogVersion = canonicalVersionAndRelease(doc.oplogVersion()); + observation.oplogFrontiers = canonicalFrontiers(doc.oplogFrontiers()); + observation.opCount = doc.opCount(); + observation.changeCount = doc.changeCount(); + } + return canonicalize(observation); + } finally { + releaseResources(nodes); + releaseResource(text); + releaseResource(tree); } - return canonicalize(observation); } export function observeDocForNative(doc) { return canonicalize({ json: doc.toJSON(), deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), - version: canonicalVersion(doc.version()), - oplogVersion: canonicalVersion(doc.oplogVersion()), + version: canonicalVersionAndRelease(doc.version()), + oplogVersion: canonicalVersionAndRelease(doc.oplogVersion()), frontiers: canonicalFrontiers(doc.frontiers()), oplogFrontiers: canonicalFrontiers(doc.oplogFrontiers()), detached: doc.isDetached(), @@ -312,147 +568,175 @@ function liveTreeNodes(tree) { } function isTreeDescendant(node, ancestorId) { - for (let parent = node.parent(); parent !== undefined; parent = parent.parent()) { - if (parent.id === ancestorId) return true; + let parent = node.parent(); + while (parent !== undefined) { + if (parent.id === ancestorId) { + releaseResource(parent); + return true; + } + const next = parent.parent(); + releaseResource(parent); + parent = next; } return false; } +function usingLiveTreeNodes(doc, action) { + return usingResource(doc.getTree("tree"), (tree) => { + const nodes = liveTreeNodes(tree); + try { + return action(tree, nodes); + } finally { + releaseResources(nodes); + } + }); +} + function applyEdit(doc, command) { switch (command.kind) { case "map-set": - return doc.getMap("map").set(command.key, command.value); - case "map-delete": { - const map = doc.getMap("map"); - const keys = map.keys(); - if (keys.length === 0) return undefined; - const key = keys.includes(command.key) ? command.key : keys.sort()[0]; - return map.delete(key); - } - case "list-insert": { - const list = doc.getList("list"); - return list.insert(modulo(command.index, list.length + 1), command.value); - } - case "list-delete": { - const list = doc.getList("list"); - if (list.length === 0) return undefined; - const index = modulo(command.index, list.length); - const length = 1 + modulo(command.length, list.length - index); - return list.delete(index, length); - } - case "text-insert": { - const text = doc.getText("text"); - const boundaries = utf16Boundaries(text.toString()); - return text.insert( - boundaries[modulo(command.index, boundaries.length)], - command.text, - ); - } - case "text-delete": { - const text = doc.getText("text"); - const { start, end } = selectedTextRange( - text, - command.index, - command.length, - false, - ); - if (start === end) return undefined; - return text.delete(start, end - start); - } - case "text-mark": { - const text = doc.getText("text"); - const { start, end } = selectedTextRange( - text, - command.index, - command.length, - false, + return usingResource(doc.getMap("map"), (map) => + map.set(command.key, command.value), ); - if (start === end) return undefined; - return text.mark({ start, end }, command.key, command.value); - } - case "text-unmark": { - const text = doc.getText("text"); - const { start, end } = selectedTextRange( - text, - command.index, - command.length, - false, + case "map-delete": + return usingResource(doc.getMap("map"), (map) => { + const keys = map.keys(); + if (keys.length === 0) return undefined; + const key = keys.includes(command.key) ? command.key : keys.sort()[0]; + return map.delete(key); + }); + case "list-insert": + return usingResource(doc.getList("list"), (list) => + list.insert(modulo(command.index, list.length + 1), command.value), ); - if (start === end) return undefined; - return text.unmark({ start, end }, command.key); - } - case "movable-insert": { - const list = doc.getMovableList("movable"); - return list.insert(modulo(command.index, list.length + 1), command.value); - } - case "movable-delete": { - const list = doc.getMovableList("movable"); - if (list.length === 0) return undefined; - const index = modulo(command.index, list.length); - const length = 1 + modulo(command.length, list.length - index); - return list.delete(index, length); - } - case "movable-set": { - const list = doc.getMovableList("movable"); - if (list.length === 0) return undefined; - const index = modulo(command.index, list.length); - if (isDeepStrictEqual(canonicalize(list.get(index)), canonicalize(command.value))) { - return undefined; - } - return list.set(index, command.value); - } - case "movable-move": { - const list = doc.getMovableList("movable"); - if (list.length < 2) return undefined; - return list.move( - modulo(command.from, list.length), - modulo(command.to, list.length), + case "list-delete": + return usingResource(doc.getList("list"), (list) => { + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + const length = 1 + modulo(command.length, list.length - index); + return list.delete(index, length); + }); + case "text-insert": + return usingResource(doc.getText("text"), (text) => { + const boundaries = utf16Boundaries(text.toString()); + return text.insert( + boundaries[modulo(command.index, boundaries.length)], + command.text, + ); + }); + case "text-delete": + return usingResource(doc.getText("text"), (text) => { + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.delete(start, end - start); + }); + case "text-mark": + return usingResource(doc.getText("text"), (text) => { + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.mark({ start, end }, command.key, command.value); + }); + case "text-unmark": + return usingResource(doc.getText("text"), (text) => { + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.unmark({ start, end }, command.key); + }); + case "movable-insert": + return usingResource(doc.getMovableList("movable"), (list) => + list.insert(modulo(command.index, list.length + 1), command.value), ); - } + case "movable-delete": + return usingResource(doc.getMovableList("movable"), (list) => { + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + const length = 1 + modulo(command.length, list.length - index); + return list.delete(index, length); + }); + case "movable-set": + return usingResource(doc.getMovableList("movable"), (list) => { + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + if ( + isDeepStrictEqual(canonicalize(list.get(index)), canonicalize(command.value)) + ) { + return undefined; + } + return list.set(index, command.value); + }); + case "movable-move": + return usingResource(doc.getMovableList("movable"), (list) => { + if (list.length < 2) return undefined; + return list.move( + modulo(command.from, list.length), + modulo(command.to, list.length), + ); + }); case "counter-increment": return command.delta === 0 ? undefined - : doc.getCounter("counter").increment(command.delta); - case "tree-create": { - const tree = doc.getTree("tree"); - const nodes = liveTreeNodes(tree); - const parent = - command.parent === null || nodes.length === 0 - ? undefined - : nodes[modulo(command.parent, nodes.length)].id; - const node = tree.createNode(parent); - node.data.set("value", command.value); - return node.id; - } - case "tree-meta-set": { - const nodes = liveTreeNodes(doc.getTree("tree")); - if (nodes.length === 0) return undefined; - return nodes[modulo(command.node, nodes.length)].data.set( - command.key, - command.value, - ); - } - case "tree-move": { - const tree = doc.getTree("tree"); - const nodes = liveTreeNodes(tree); - if (nodes.length === 0) return undefined; - const node = nodes[modulo(command.node, nodes.length)]; - const candidates = nodes.filter( - (candidate) => candidate.id !== node.id && !isTreeDescendant(candidate, node.id), - ); - const parent = - command.parent === null || candidates.length === 0 - ? undefined - : candidates[modulo(command.parent, candidates.length)]; - if (node.parent()?.id === parent?.id) return undefined; - return tree.move(node.id, parent?.id); - } - case "tree-delete": { - const tree = doc.getTree("tree"); - const nodes = liveTreeNodes(tree); - if (nodes.length === 0) return undefined; - return tree.delete(nodes[modulo(command.node, nodes.length)].id); - } + : usingResource(doc.getCounter("counter"), (counter) => + counter.increment(command.delta), + ); + case "tree-create": + return usingLiveTreeNodes(doc, (tree, nodes) => { + const parent = + command.parent === null || nodes.length === 0 + ? undefined + : nodes[modulo(command.parent, nodes.length)].id; + const node = tree.createNode(parent); + try { + usingResource(node.data, (data) => data.set("value", command.value)); + return node.id; + } finally { + releaseResource(node); + } + }); + case "tree-meta-set": + return usingLiveTreeNodes(doc, (_tree, nodes) => { + if (nodes.length === 0) return undefined; + return usingResource(nodes[modulo(command.node, nodes.length)].data, (data) => + data.set(command.key, command.value), + ); + }); + case "tree-move": + return usingLiveTreeNodes(doc, (tree, nodes) => { + if (nodes.length === 0) return undefined; + const node = nodes[modulo(command.node, nodes.length)]; + const candidates = nodes.filter( + (candidate) => + candidate.id !== node.id && !isTreeDescendant(candidate, node.id), + ); + const parent = + command.parent === null || candidates.length === 0 + ? undefined + : candidates[modulo(command.parent, candidates.length)]; + const currentParent = node.parent(); + const currentParentId = currentParent?.id; + releaseResource(currentParent); + if (currentParentId === parent?.id) return undefined; + return tree.move(node.id, parent?.id); + }); + case "tree-delete": + return usingLiveTreeNodes(doc, (tree, nodes) => { + if (nodes.length === 0) return undefined; + return tree.delete(nodes[modulo(command.node, nodes.length)].id); + }); default: throw new RangeError(`unsupported edit command ${command.kind}`); } @@ -464,20 +748,24 @@ function createDoc(engine, peer, initializeRoots = true) { doc.setRecordTimestamp(false); doc.setChangeMergeInterval(0); if (initializeRoots) { - doc.getMap("map"); - doc.getList("list"); - doc.getText("text"); - doc.getMovableList("movable"); - doc.getCounter("counter"); - doc.getTree("tree").enableFractionalIndex(0); + releaseResource(doc.getMap("map")); + releaseResource(doc.getList("list")); + releaseResource(doc.getText("text")); + releaseResource(doc.getMovableList("movable")); + releaseResource(doc.getCounter("counter")); + usingResource(doc.getTree("tree"), (tree) => tree.enableFractionalIndex(0)); } return doc; } function exportFor(doc, target, mode) { - return mode === "snapshot" - ? doc.export({ mode: "snapshot" }) - : doc.export({ mode: "update", from: target.oplogVersion() }); + if (mode === "snapshot") return doc.export({ mode: "snapshot" }); + const from = target.oplogVersion(); + try { + return doc.export({ mode: "update", from }); + } finally { + releaseResource(from); + } } function compare(label, left, right) { @@ -506,8 +794,7 @@ class EngineWorld { this.eventQueues = this.docs.map(() => []); this.subscriptions = this.docs.map((doc, peer) => doc.subscribe((batch) => { - const normalized = normalizeEventBatch(batch, doc); - if (normalized.events.length > 0) this.eventQueues[peer].push(normalized); + this.eventQueues[peer].push(batch); }), ); } @@ -588,8 +875,12 @@ class EngineWorld { ? doc.export({ mode: "snapshot" }) : doc.export({ mode: "update" }); const imported = createDoc(this.engine, undefined, false); - imported.import(bytes); - return observeDoc(imported); + try { + imported.import(bytes); + return observeDoc(imported); + } finally { + releaseResource(imported); + } }); } @@ -599,7 +890,20 @@ class EngineWorld { drainEvents() { this.engine.callPendingEvents?.(); - return this.eventQueues.map((queue) => queue.splice(0)); + return this.eventQueues.map((queue) => + queue + .splice(0) + .map(normalizeEventBatch) + .filter((batch) => batch.events.length > 0), + ); + } + + dispose() { + for (const subscription of this.subscriptions) { + if (typeof subscription === "function") subscription(); + else releaseResource(subscription); + } + releaseResources(this.docs); } } @@ -616,13 +920,17 @@ function isEditCommand(kind) { export function runSingleScenario(engine, scenario, externalBlobs) { const world = new EngineWorld(engine, scenario, externalBlobs); - const results = []; - for (const command of scenario.commands) results.push(world.execute(command)); - return { - observations: world.finalNativeObservations(), - transportBlobs: world.transportBlobs, - results: canonicalize(results), - }; + try { + const results = []; + for (const command of scenario.commands) results.push(world.execute(command)); + return { + observations: world.finalNativeObservations(), + transportBlobs: world.transportBlobs, + results: canonicalize(results), + }; + } finally { + world.dispose(); + } } export function runMalformedImportChecks(wasmEngine, jsEngine) { @@ -633,24 +941,30 @@ export function runMalformedImportChecks(wasmEngine, jsEngine) { let cases = 0; for (const [producerName, producer] of engines) { const source = createDoc(producer, 0); - source.getMap("map").set("payload", { nested: [1, true, "😀"] }); - source.getText("text").insert(0, "malformed 😀 文"); - source.getList("list").push("item"); - source.commit({ message: "malformed-seed" }); - for (const mode of ["update", "snapshot"]) { - const blob = source.export({ mode }); - for (const [mutation, bytes] of malformedVariants(blob)) { - const outcomes = engines.map(([consumerName, engine]) => [ - consumerName, - malformedOutcome(engine, bytes), - ]); - compare( - `malformed import mismatch: producer=${producerName} mode=${mode} mutation=${mutation}`, - outcomes[0][1], - outcomes[1][1], - ); - cases += 1; + try { + usingResource(source.getMap("map"), (map) => + map.set("payload", { nested: [1, true, "😀"] }), + ); + usingResource(source.getText("text"), (text) => text.insert(0, "malformed 😀 文")); + usingResource(source.getList("list"), (list) => list.push("item")); + source.commit({ message: "malformed-seed" }); + for (const mode of ["update", "snapshot"]) { + const blob = source.export({ mode }); + for (const [mutation, bytes] of malformedVariants(blob)) { + const outcomes = engines.map(([consumerName, engine]) => [ + consumerName, + malformedOutcome(engine, bytes), + ]); + compare( + `malformed import mismatch: producer=${producerName} mode=${mode} mutation=${mutation}`, + outcomes[0][1], + outcomes[1][1], + ); + cases += 1; + } } + } finally { + releaseResource(source); } } return cases; @@ -678,18 +992,22 @@ function flippedByte(blob, index) { function malformedOutcome(engine, bytes) { const target = createDoc(engine, 97); - target.getMap("map").set("sentinel", "unchanged"); - target.getText("text").insert(0, "stable"); - target.commit({ message: "malformed-target" }); - const before = observeDoc(target); try { - target.import(bytes); - return { accepted: true, observation: observeDoc(target) }; - } catch { - return { - accepted: false, - atomic: isDeepStrictEqual(before, observeDoc(target)), - }; + usingResource(target.getMap("map"), (map) => map.set("sentinel", "unchanged")); + usingResource(target.getText("text"), (text) => text.insert(0, "stable")); + target.commit({ message: "malformed-target" }); + const before = observeDoc(target); + try { + target.import(bytes); + return { accepted: true, observation: observeDoc(target) }; + } catch { + return { + accepted: false, + atomic: isDeepStrictEqual(before, observeDoc(target)), + }; + } + } finally { + releaseResource(target); } } @@ -702,69 +1020,74 @@ export function runDifferentialScenario( const wasm = new EngineWorld(wasmEngine, scenario); const js = new EngineWorld(jsEngine, scenario); - for (let index = 0; index < scenario.commands.length; index += 1) { - const command = scenario.commands[index]; - let wasmResult; - let jsResult; - let compareEvents = true; - if (command.kind === "deliver") { - const wasmSlot = wasm.slots.get(modulo(command.slot, 8)); - const jsSlot = js.slots.get(modulo(command.slot, 8)); - if (wasmSlot === undefined || jsSlot === undefined) { - wasmResult = { ok: true, value: "empty-slot" }; - jsResult = { ok: true, value: "empty-slot" }; - } else { - compareEvents = strict || wasmSlot.mode !== "snapshot"; - const copies = 1 + modulo(command.copies, 3); - const wasmStatuses = []; - const jsStatuses = []; - for (let copy = 0; copy < copies; copy += 1) { - wasmStatuses.push( - canonicalize(wasm.docs[wasmSlot.targetIndex].import(jsSlot.blob)), - ); - jsStatuses.push( - canonicalize(js.docs[jsSlot.targetIndex].import(wasmSlot.blob)), - ); + try { + for (let index = 0; index < scenario.commands.length; index += 1) { + const command = scenario.commands[index]; + let wasmResult; + let jsResult; + let compareEvents = true; + if (command.kind === "deliver") { + const wasmSlot = wasm.slots.get(modulo(command.slot, 8)); + const jsSlot = js.slots.get(modulo(command.slot, 8)); + if (wasmSlot === undefined || jsSlot === undefined) { + wasmResult = { ok: true, value: "empty-slot" }; + jsResult = { ok: true, value: "empty-slot" }; + } else { + compareEvents = strict || wasmSlot.mode !== "snapshot"; + const copies = 1 + modulo(command.copies, 3); + const wasmStatuses = []; + const jsStatuses = []; + for (let copy = 0; copy < copies; copy += 1) { + wasmStatuses.push( + canonicalize(wasm.docs[wasmSlot.targetIndex].import(jsSlot.blob)), + ); + jsStatuses.push( + canonicalize(js.docs[jsSlot.targetIndex].import(wasmSlot.blob)), + ); + } + wasmResult = { ok: true, value: wasmStatuses }; + jsResult = { ok: true, value: jsStatuses }; } - wasmResult = { ok: true, value: wasmStatuses }; - jsResult = { ok: true, value: jsStatuses }; + } else if (command.kind === "roundtrip") { + const peer = peerIndex(command.peer, wasm.docs.length); + wasmResult = crossRoundtrip( + wasmEngine, + jsEngine, + wasm.docs[peer], + js.docs[peer], + command.mode, + ); + jsResult = wasmResult; + } else { + wasmResult = wasm.execute(command); + jsResult = js.execute(command); } - } else if (command.kind === "roundtrip") { - const peer = peerIndex(command.peer, wasm.docs.length); - wasmResult = crossRoundtrip( - wasmEngine, - jsEngine, - wasm.docs[peer], - js.docs[peer], - command.mode, - ); - jsResult = wasmResult; - } else { - wasmResult = wasm.execute(command); - jsResult = js.execute(command); - } - compare( - `command result mismatch at ${index}: ${JSON.stringify(command)}`, - wasmResult, - jsResult, - ); - const wasmEvents = wasm.drainEvents(); - const jsEvents = js.drainEvents(); - if (compareEvents) { compare( - `event mismatch at command ${index}: ${JSON.stringify(command)}`, - wasmEvents, - jsEvents, - ); - } - for (let peer = 0; peer < scenario.peerCount; peer += 1) { - compare( - `document mismatch at command ${index}, peer ${peer}: ${JSON.stringify(command)}`, - observeDoc(wasm.docs[peer]), - observeDoc(js.docs[peer]), + `command result mismatch at ${index}: ${JSON.stringify(command)}`, + wasmResult, + jsResult, ); + const wasmEvents = wasm.drainEvents(); + const jsEvents = js.drainEvents(); + if (compareEvents) { + compare( + `event mismatch at command ${index}: ${JSON.stringify(command)}`, + wasmEvents, + jsEvents, + ); + } + for (let peer = 0; peer < scenario.peerCount; peer += 1) { + compare( + `document mismatch at command ${index}, peer ${peer}: ${JSON.stringify(command)}`, + observeDoc(wasm.docs[peer]), + observeDoc(js.docs[peer]), + ); + } } + } finally { + wasm.dispose(); + js.dispose(); } } @@ -785,8 +1108,12 @@ function crossRoundtrip(wasmEngine, jsEngine, wasmDoc, jsDoc, mode) { [jsEngine, jsBytes], ].map(([engine, bytes]) => { const doc = createDoc(engine, undefined, false); - doc.import(bytes); - return observeDoc(doc); + try { + doc.import(bytes); + return observeDoc(doc); + } finally { + releaseResource(doc); + } }); for (let index = 1; index < variants.length; index += 1) { compare(`roundtrip variant ${index} mismatch`, variants[0], variants[index]); diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 69e5fa726..aaaec1d52 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -2124,6 +2124,11 @@ export interface TreeNodeRecord { readonly data: LoroMap; } +const DELETED_TREE_ROOT_ID: CodecId = { + peer: 18_446_744_073_709_551_615n, + counter: 2_147_483_647, +}; + interface TreeJsonValue { readonly id: TreeID; readonly parent: TreeID | null; @@ -2139,11 +2144,9 @@ export class LoroTree< readonly _nodes = new Map(); // Parent-keyed children indexes, keyed numerically (peer -> counter) so tree // reads/writes never stringify ids. Root children live in `_rootChildren`. - readonly _children = new Map< - bigint, - Map> - >(); + readonly _children = new Map>>(); private _rootChildren: OrderedIndex | undefined; + private _deletedRoots: OrderedIndex | undefined; _fractionalIndexEnabled = true; kind(): "Tree" { @@ -2176,7 +2179,11 @@ export class LoroTree< } isNodeDeleted(target: TreeID): boolean { - return this._nodes.get(target)?.deleted ?? false; + const node = this._nodes.get(target); + if (node === undefined) { + throw new RangeError(`tree node ${target} does not exist`); + } + return this._isRecordDeleted(node); } enableFractionalIndex(jitter = 0): void { @@ -2200,13 +2207,17 @@ export class LoroTree< } getNodes(options: { withDeleted?: boolean } = {}): LoroTreeNode[] { - const output: LoroTreeNode[] = []; - for (const record of this._nodes.values()) { - if (options.withDeleted === true || !record.deleted) { - output.push(new LoroTreeNode(this, record.id)); - } + const records = this._nodesUnder(this._childrenOf(undefined)); + if (options.withDeleted === true) { + records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); } - return output; + return records.map((record) => new LoroTreeNode(this, record.id)); + } + + _recordsForSnapshot(): TreeNodeRecord[] { + const records = this._nodesUnder(this._childrenOf(undefined)); + records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); + return records; } nodes(): LoroTreeNode[] { @@ -2260,9 +2271,7 @@ export class LoroTree< } _rootJsonValuesRange(start: number, end: number): TreeJsonValue[] { - return ( - this._childrenIndexFor(undefined)?.valuesRange(start, end) ?? [] - ).map( + return (this._childrenIndexFor(undefined)?.valuesRange(start, end) ?? []).map( (record, offset) => this._recordToValue(record, start + offset) as TreeJsonValue, ); } @@ -2310,7 +2319,12 @@ export class LoroTree< _setRecord(record: TreeNodeRecord): void { this._nodes.set(formatTreeId(record.id), record); record.data._setParentBinding(this, { kind: "tree", record }); - if (!record.deleted) this._childrenIndex(record.parent).add(record); + if (record.deleted) { + record.parent = DELETED_TREE_ROOT_ID; + this._deletedRootsIndex().add(record); + } else { + this._childrenIndex(record.parent).add(record); + } } _updateRecord( @@ -2320,7 +2334,11 @@ export class LoroTree< writer: LastWriter, lastMoveId: CodecId, ): void { - if (!record.deleted) this._childrenIndexFor(record.parent)?.delete(record); + if (record.deleted) { + this._deletedRoots?.delete(record); + } else { + this._childrenIndexFor(record.parent)?.delete(record); + } record.parent = parent; record.position = position; record.deleted = false; @@ -2329,22 +2347,49 @@ export class LoroTree< this._childrenIndex(parent).add(record); } - _deleteRecord(record: TreeNodeRecord, writer: LastWriter): void { - if (!record.deleted) this._childrenIndexFor(record.parent)?.delete(record); + _deleteRecord(record: TreeNodeRecord, writer: LastWriter, lastMoveId: CodecId): void { + if (record.deleted) { + this._deletedRoots?.delete(record); + } else { + this._childrenIndexFor(record.parent)?.delete(record); + } + record.parent = DELETED_TREE_ROOT_ID; + record.position = Uint8Array.of(0x80); record.deleted = true; record.writer = writer; + record.lastMoveId = lastMoveId; + this._deletedRootsIndex().add(record); } _removeRecord(record: TreeNodeRecord): void { - if (!record.deleted) this._childrenIndexFor(record.parent)?.delete(record); + if (record.deleted) { + this._deletedRoots?.delete(record); + } else { + this._childrenIndexFor(record.parent)?.delete(record); + } record.deleted = true; this._nodes.delete(formatTreeId(record.id)); } _indexOf(record: TreeNodeRecord): number { + if (record.deleted) return -1; return this._childrenIndexFor(record.parent)?.indexOf(record) ?? -1; } + _isRecordDeleted(record: TreeNodeRecord): boolean { + const visited = new Set(); + let current: TreeNodeRecord | undefined = record; + while (current !== undefined) { + if (current.deleted) return true; + if (current.parent === undefined) return false; + const key = formatTreeId(current.parent); + if (visited.has(key)) return true; + visited.add(key); + current = this._nodes.get(key); + } + return true; + } + _recordToValue( record: TreeNodeRecord, index = this._indexOf(record), @@ -2405,6 +2450,18 @@ export class LoroTree< this._nodes.clear(); this._children.clear(); this._rootChildren = undefined; + this._deletedRoots = undefined; + } + + private _nodesUnder(roots: readonly TreeNodeRecord[]): TreeNodeRecord[] { + const output: TreeNodeRecord[] = []; + const queue = roots.slice(); + for (let index = 0; index < queue.length; index += 1) { + const record = queue[index]!; + output.push(record); + queue.push(...this._childrenOf(record.id)); + } + return output; } private _childrenIndexFor( @@ -2431,6 +2488,10 @@ export class LoroTree< } return index; } + + private _deletedRootsIndex(): OrderedIndex { + return (this._deletedRoots ??= new OrderedIndex(compareTreeRecords)); + } } export class LoroTreeNode = Record> { @@ -2462,11 +2523,15 @@ export class LoroTreeNode = Record): void { const parent = target.parent(); - this.move(parent, target.index() + 1); + const index = target.index(); + if (index === undefined) throw new RangeError(`tree node ${target.id} is deleted`); + this.move(parent, index + 1); } moveBefore(target: LoroTreeNode): void { - this.move(target.parent(), target.index()); + const index = target.index(); + if (index === undefined) throw new RangeError(`tree node ${target.id} is deleted`); + this.move(target.parent(), index); } parent(): LoroTreeNode | undefined { @@ -2484,19 +2549,21 @@ export class LoroTreeNode = Record 0xffff ? 2 : 1; } container.delete(prefixLength, currentLength - prefixLength - suffixLength); - container.insert( - prefixLength, - target.slice(prefixLength, targetLength - suffixLength), - ); + container.insert(prefixLength, target.slice(prefixLength, targetLength - suffixLength)); } diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index c5c2e64d7..1970be88a 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -236,6 +236,7 @@ type PendingEventState = readonly value: unknown; } >; + readonly changed: Set; } | { readonly kind: "counter"; readonly before: number } | { @@ -660,6 +661,9 @@ export class LoroDoc = Record = Record = Record = Record { + const afterRecord = container._entries.get(content.key); + const afterVisible = afterRecord !== undefined && !afterRecord.deleted; + const afterValue = afterVisible + ? cloneRuntimeValue(afterRecord.value) + : undefined; + if ( + beforeVisible !== afterVisible || + !eventValuesEqual(beforeValue, afterValue) + ) { + state.changed.add(content.key); + } + }; } } else if (container instanceof LoroTree) { if ( @@ -2903,7 +2927,7 @@ export class LoroDoc = Record = Record(), + changed: new Set(), }; recording.eventStates.set(container.id, state); return state; @@ -3021,6 +3046,7 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record Number(left.deleted) - Number(right.deleted), - ); + // Rust's fast-tree decoder historically accepted the canonical snapshot + // order: live roots/subtrees first, then deleted roots/subtrees, with each + // sibling group ordered by fractional position and last-move writer. + // Preserve that order so snapshots remain byte-level interoperable and so + // parent-local ordered insertion never sees an insertion-order permutation. + const records = container._recordsForSnapshot(); const peers: bigint[] = []; const peerIndices = new Map(); const peerIndex = (peer: bigint): bigint => { @@ -5478,19 +5515,18 @@ export class LoroDoc = Record = Record (incomingVersion.get(peer) ?? 0)) { + incomingVersion.set(peer, counter); + } + } + } + + // Rust calculates a concurrent import diff by retreating to the common + // ancestor and replaying both branches. Tree operations from the local-only + // branch can therefore produce visible reindex moves even though those + // operations are not present in the imported update. Capture those subjects + // so the incremental TS merge reports the same net transition. + for (const { change } of this.#recordsInVersionRange( + incomingVersion, + beforeVersion, + )) { + for (const operation of change.operations) { + const content = operation.content; + if ( + content.type !== "tree-create" && + content.type !== "tree-move" && + content.type !== "tree-delete" + ) { + continue; + } + const container = this.#containers.get(this.#containerKey(operation.container)); + if (!(container instanceof LoroTree)) continue; + const state = this.#treeEventState(recording, container); + const nodeId = formatTreeId(content.subject); + if (state.originals.has(nodeId)) continue; + const record = container._nodes.get(nodeId); + state.originals.set( + nodeId, + record === undefined || container._isRecordDeleted(record) + ? undefined + : this.#treeEventNode(container, record), + ); + } + } + } + #captureContainerEventValues(ids: ReadonlySet): Map { const values = new Map(); for (const id of ids) { @@ -5897,12 +5987,34 @@ export class LoroDoc = Record = new Map(), ): void { if (changed.size === 0 || !this.#hasEventSubscribers()) return; - const events: LoroEvent[] = []; + const expandedChanged = new Set(changed); + const revived = new Set(); + const calculatedDiffs = new Map(preparedDiffs); for (const id of changed) { const container = this.#containers.get(id); - if (container === undefined) continue; + if (!(container instanceof LoroTree) || this._isContainerDeleted(container)) { + continue; + } + const diff = + calculatedDiffs.get(id) ?? containerDiff(container, beforeValues.get(id)); + calculatedDiffs.set(id, diff); + if (diff.type !== "tree") continue; + for (const item of diff.diff) { + if (item.action !== "create") continue; + const record = container._nodes.get(item.target); + if (record === undefined || container._isRecordDeleted(record)) continue; + expandedChanged.add(record.data.id); + revived.add(record.data.id); + } + } + + const events: LoroEvent[] = []; + for (const id of expandedChanged) { + const container = this.#containers.get(id); + if (container === undefined || this._isContainerDeleted(container)) continue; const diff = - preparedDiffs.get(id) ?? containerDiff(container, beforeValues.get(id)); + calculatedDiffs.get(id) ?? + containerDiff(container, revived.has(id) ? undefined : beforeValues.get(id)); if (isEmptyContainerDiff(diff)) continue; events.push({ target: id as ContainerID, diff, path: containerPath(container) }); } @@ -7814,7 +7926,7 @@ function containerEventValue(container: LoroContainer): unknown { return container.getNodes().map((node) => ({ id: node.id, parent: node.parent()?.id, - index: node.index(), + index: node.index()!, fractionalIndex: node.fractionalIndex() ?? "", })) satisfies TreeEventNode[]; } diff --git a/loro-js/tests/event-diff.test.ts b/loro-js/tests/event-diff.test.ts index 1508556cb..224b9d0e3 100644 --- a/loro-js/tests/event-diff.test.ts +++ b/loro-js/tests/event-diff.test.ts @@ -238,6 +238,46 @@ describe("incremental event diffs", () => { ], }); }); + + test("reports local tree reindexing during a concurrent import", () => { + const incoming = new LoroDoc(); + incoming.setPeerId(1); + const incomingNode = incoming.getTree("tree").createNode(); + incoming.commit(); + + const target = new LoroDoc(); + target.setPeerId(2); + const localNode = target.getTree("tree").createNode(); + target.commit(); + let batch: LoroEventBatch | undefined; + target.subscribe((event) => { + batch = event; + }); + + target.import(incoming.export({ mode: "update" })); + + expect(batch?.events[0]?.diff).toEqual({ + type: "tree", + diff: [ + { + target: localNode.id, + action: "move", + parent: undefined, + index: 1, + fractionalIndex: localNode.fractionalIndex(), + oldParent: undefined, + oldIndex: 0, + }, + { + target: incomingNode.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: incomingNode.fractionalIndex(), + }, + ], + }); + }); }); function applyTextDelta(original: string, delta: readonly Delta[]): string { diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 752d2b4be..4c75e6177 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -527,6 +527,73 @@ describe("loro-wasm-compatible runtime", () => { }); }); + test("emits the final map value after a delete and restore in one commit", () => { + const doc = new LoroDoc(); + doc.setPeerId(2); + const map = doc.getMap("map"); + map.set("title", {}); + doc.commit(); + + const batches: LoroEventBatch[] = []; + doc.subscribe((event) => batches.push(event)); + map.delete("title"); + map.set("title", {}); + doc.attach(); + + expect(batches).toHaveLength(1); + expect(batches[0]?.events).toEqual([ + { + target: map.id, + diff: { type: "map", updated: { title: {} } }, + path: ["map"], + }, + ]); + }); + + test("preserves visible map transitions but suppresses same-value import winners", () => { + const source = new LoroDoc(); + source.setPeerId(2); + const sourceMap = source.getMap("map"); + sourceMap.set("title", {}); + source.commit(); + + const target = new LoroDoc(); + target.setPeerId(1); + target.import(source.export({ mode: "update" })); + const batches: LoroEventBatch[] = []; + target.subscribe((event) => batches.push(event)); + + sourceMap.delete("title"); + sourceMap.set("title", {}); + source.commit(); + target.import(source.export({ mode: "update" })); + expect(batches.map((batch) => batch.events[0]?.diff)).toEqual([ + { type: "map", updated: { title: {} } }, + ]); + + const concurrent = new LoroDoc(); + concurrent.setPeerId(3); + concurrent.getMap("map").set("title", {}); + concurrent.commit(); + batches.length = 0; + target.import(concurrent.export({ mode: "update" })); + expect(batches).toEqual([]); + }); + + test("does not emit changes for tree metadata deleted in the same commit", () => { + const doc = new LoroDoc(); + const tree = doc.getTree("tree"); + const batches: LoroEventBatch[] = []; + doc.subscribe((event) => batches.push(event)); + + const node = tree.createNode(); + node.data.set("value", []); + tree.delete(node.id); + doc.commit(); + + expect(batches).toEqual([]); + }); + test("calculates raw and JSON-compatible diffs without moving document state", () => { const doc = new LoroDoc(); const map = doc.getMap("map"); @@ -1770,15 +1837,60 @@ describe("loro-wasm-compatible runtime", () => { expect(tree.isFractionalIndexEnabled()).toBe(true); expect(() => tree.enableFractionalIndex(256)).toThrow(/unsigned byte/u); - const deletedFractionalIndex = child.fractionalIndex(); + const lastMoveBeforeDelete = child.getLastMoveId(); tree.delete(child.id); expect(tree.isNodeDeleted(child.id)).toBe(true); + expect(() => tree.isNodeDeleted("999@999")).toThrow(/does not exist/u); expect(tree.getNodes({ withDeleted: true })).toHaveLength(2); expect(root.toJSON()).toMatchObject({ parent: undefined, fractionalIndex: "80" }); + expect(child.parent()?.id).toBe("2147483647@18446744073709551615"); + expect(child.index()).toBeUndefined(); + expect(child.fractionalIndex()).toBeUndefined(); + expect(child.getLastMoveId()).not.toEqual(lastMoveBeforeDelete); expect(child.toJSON()).toMatchObject({ parent: "2147483647@18446744073709551615", index: 0, - fractionalIndex: deletedFractionalIndex, + fractionalIndex: "80", + }); + }); + + test("keeps deleted tree subtrees out of the live forest and snapshots them", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + tree.enableFractionalIndex(0); + const liveRoot = tree.createNode(); + const deletedRoot = tree.createNode(); + const deletedChild = deletedRoot.createNode(); + + expect(deletedRoot.toJSON().fractionalIndex).toBe("8180"); + tree.delete(deletedRoot.id); + expect(tree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); + expect(tree.getNodes({ withDeleted: true }).map((node) => node.id)).toEqual([ + liveRoot.id, + deletedRoot.id, + deletedChild.id, + ]); + expect(tree.isNodeDeleted(deletedChild.id)).toBe(true); + expect(deletedChild.isDeleted()).toBe(true); + expect(deletedChild.data.isDeleted()).toBe(true); + expect(deletedRoot.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", + }); + + doc.commit(); + const restored = new LoroDoc(); + restored.import(doc.export({ mode: "snapshot" })); + const restoredTree = restored.getTree("tree"); + expect(restoredTree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); + expect(restoredTree.isNodeDeleted(deletedRoot.id)).toBe(true); + expect(restoredTree.isNodeDeleted(deletedChild.id)).toBe(true); + expect(restoredTree.getNodeByID(deletedRoot.id)?.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", }); }); diff --git a/loro-js/tests/version-transition.test.ts b/loro-js/tests/version-transition.test.ts index 2a254f518..6a8f61667 100644 --- a/loro-js/tests/version-transition.test.ts +++ b/loro-js/tests/version-transition.test.ts @@ -4,6 +4,75 @@ import { LoroDoc } from "../src/runtime/document"; import type { LoroEventBatch } from "../src/runtime/types"; describe("indexed version transitions", () => { + test("reports tree checkout deletions at their original sibling indices", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const emptyFrontiers = doc.frontiers(); + const tree = doc.getTree("tree"); + const first = tree.createNode(); + const second = tree.createNode(); + doc.commit(); + let batch: LoroEventBatch | undefined; + doc.subscribe((event) => { + batch = event; + }); + + doc.checkout(emptyFrontiers); + + expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ + type: "tree", + diff: [ + { + target: first.id, + action: "delete", + oldParent: undefined, + oldIndex: 0, + }, + { + target: second.id, + action: "delete", + oldParent: undefined, + oldIndex: 1, + }, + ], + }); + }); + + test("revives tree metadata events when checkout restores a node", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const node = tree.createNode(); + node.data.set("value", {}); + doc.commit(); + const liveFrontiers = doc.frontiers(); + tree.delete(node.id); + doc.commit(); + let batch: LoroEventBatch | undefined; + doc.subscribe((event) => { + batch = event; + }); + + doc.checkout(liveFrontiers); + + expect(batch?.events.find(({ target }) => target === node.data.id)?.diff).toEqual({ + type: "map", + updated: { value: {} }, + }); + expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ + type: "tree", + diff: [ + { + target: node.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: node.fractionalIndex(), + }, + ], + }); + }); + test("retreats and restores indexed container state", () => { const doc = new LoroDoc(); doc.setPeerId(1); From 5b12501dbc745512b19cc1b42405565481d37413 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 20 Jul 2026 01:44:56 +0800 Subject: [PATCH 3/5] chore: add generated loro-js JavaScript --- loro-js/src/codec/bytes.js | 235 + loro-js/src/codec/change-block-codec.js | 789 ++ loro-js/src/codec/change-block-tables.js | 375 + loro-js/src/codec/change-block.js | 38 + loro-js/src/codec/change-value.js | 375 + loro-js/src/codec/container-id.js | 225 + loro-js/src/codec/document.js | 115 + loro-js/src/codec/errors.js | 24 + loro-js/src/codec/id.js | 53 + loro-js/src/codec/index.js | 20 + loro-js/src/codec/leb128.js | 149 + loro-js/src/codec/lz4.js | 307 + loro-js/src/codec/position-arena.js | 67 + loro-js/src/codec/postcard.js | 172 + loro-js/src/codec/serde-columnar.js | 902 +++ loro-js/src/codec/sstable.js | 261 + loro-js/src/codec/state-snapshot.js | 731 ++ loro-js/src/codec/types.js | 8 + loro-js/src/codec/value.js | 156 + loro-js/src/codec/version.js | 60 + loro-js/src/codec/xxhash32.js | 68 + loro-js/src/index.js | 1 + loro-js/src/runtime/containers.js | 2597 +++++++ loro-js/src/runtime/document.js | 6813 +++++++++++++++++ loro-js/src/runtime/ephemeral.js | 560 ++ loro-js/src/runtime/event-diff.js | 371 + loro-js/src/runtime/fractional-index.js | 92 + loro-js/src/runtime/ids.js | 151 + loro-js/src/runtime/index.js | 8 + loro-js/src/runtime/json-updates.js | 60 + loro-js/src/runtime/jsonpath.js | 833 ++ loro-js/src/runtime/mergeable.js | 124 + loro-js/src/runtime/ordered-index.js | 303 + .../src/runtime/sequence-deletion-index.js | 227 + loro-js/src/runtime/sequence-index.js | 3173 ++++++++ loro-js/src/runtime/text-style-index.js | 336 + loro-js/src/runtime/types.js | 1 + loro-js/src/runtime/undo.js | 274 + loro-js/src/runtime/version-vector.js | 112 + loro-js/tests/change-block-codec.test.js | 133 + loro-js/tests/change-block-tables.test.js | 69 + loro-js/tests/change-value.test.js | 106 + loro-js/tests/document.test.js | 40 + loro-js/tests/event-diff.test.js | 290 + loro-js/tests/fugue-index.test.js | 21 + loro-js/tests/indexes.test.js | 620 ++ loro-js/tests/lz4.test.js | 65 + loro-js/tests/primitives.test.js | 54 + loro-js/tests/runtime.test.js | 1708 +++++ loro-js/tests/rust-interop.test.js | 176 + loro-js/tests/semantic-codecs.test.js | 110 + loro-js/tests/serde-columnar.test.js | 84 + loro-js/tests/sstable.test.js | 33 + loro-js/tests/state-snapshot.test.js | 141 + loro-js/tests/version-transition.test.js | 790 ++ loro-js/vite.config.js | 39 + 56 files changed, 25645 insertions(+) create mode 100644 loro-js/src/codec/bytes.js create mode 100644 loro-js/src/codec/change-block-codec.js create mode 100644 loro-js/src/codec/change-block-tables.js create mode 100644 loro-js/src/codec/change-block.js create mode 100644 loro-js/src/codec/change-value.js create mode 100644 loro-js/src/codec/container-id.js create mode 100644 loro-js/src/codec/document.js create mode 100644 loro-js/src/codec/errors.js create mode 100644 loro-js/src/codec/id.js create mode 100644 loro-js/src/codec/index.js create mode 100644 loro-js/src/codec/leb128.js create mode 100644 loro-js/src/codec/lz4.js create mode 100644 loro-js/src/codec/position-arena.js create mode 100644 loro-js/src/codec/postcard.js create mode 100644 loro-js/src/codec/serde-columnar.js create mode 100644 loro-js/src/codec/sstable.js create mode 100644 loro-js/src/codec/state-snapshot.js create mode 100644 loro-js/src/codec/types.js create mode 100644 loro-js/src/codec/value.js create mode 100644 loro-js/src/codec/version.js create mode 100644 loro-js/src/codec/xxhash32.js create mode 100644 loro-js/src/index.js create mode 100644 loro-js/src/runtime/containers.js create mode 100644 loro-js/src/runtime/document.js create mode 100644 loro-js/src/runtime/ephemeral.js create mode 100644 loro-js/src/runtime/event-diff.js create mode 100644 loro-js/src/runtime/fractional-index.js create mode 100644 loro-js/src/runtime/ids.js create mode 100644 loro-js/src/runtime/index.js create mode 100644 loro-js/src/runtime/json-updates.js create mode 100644 loro-js/src/runtime/jsonpath.js create mode 100644 loro-js/src/runtime/mergeable.js create mode 100644 loro-js/src/runtime/ordered-index.js create mode 100644 loro-js/src/runtime/sequence-deletion-index.js create mode 100644 loro-js/src/runtime/sequence-index.js create mode 100644 loro-js/src/runtime/text-style-index.js create mode 100644 loro-js/src/runtime/types.js create mode 100644 loro-js/src/runtime/undo.js create mode 100644 loro-js/src/runtime/version-vector.js create mode 100644 loro-js/tests/change-block-codec.test.js create mode 100644 loro-js/tests/change-block-tables.test.js create mode 100644 loro-js/tests/change-value.test.js create mode 100644 loro-js/tests/document.test.js create mode 100644 loro-js/tests/event-diff.test.js create mode 100644 loro-js/tests/fugue-index.test.js create mode 100644 loro-js/tests/indexes.test.js create mode 100644 loro-js/tests/lz4.test.js create mode 100644 loro-js/tests/primitives.test.js create mode 100644 loro-js/tests/runtime.test.js create mode 100644 loro-js/tests/rust-interop.test.js create mode 100644 loro-js/tests/semantic-codecs.test.js create mode 100644 loro-js/tests/serde-columnar.test.js create mode 100644 loro-js/tests/sstable.test.js create mode 100644 loro-js/tests/state-snapshot.test.js create mode 100644 loro-js/tests/version-transition.test.js create mode 100644 loro-js/vite.config.js diff --git a/loro-js/src/codec/bytes.js b/loro-js/src/codec/bytes.js new file mode 100644 index 000000000..518dded7e --- /dev/null +++ b/loro-js/src/codec/bytes.js @@ -0,0 +1,235 @@ +import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; +const U8_MAX = 0xff; +const U16_MAX = 0xffff; +const U32_MAX = 4294967295; +const U64_MAX = 0xffffffffffffffffn; +export class ByteReader { + bytes; + #offset; + #end; + constructor(bytes, offset = 0, length = bytes.length - offset) { + decodeAssert(Number.isSafeInteger(offset) && offset >= 0, "invalid reader offset"); + decodeAssert(Number.isSafeInteger(length) && length >= 0, "invalid reader length"); + decodeAssert(offset + length <= bytes.length, "reader range is out of bounds"); + this.bytes = bytes; + this.#offset = offset; + this.#end = offset + length; + } + get position() { + return this.#offset; + } + get remaining() { + return this.#end - this.#offset; + } + readU8() { + this.require(1); + const value = this.bytes[this.#offset]; + this.#offset += 1; + return value; + } + readU16LE() { + const a = this.readU8(); + const b = this.readU8(); + return a | (b << 8); + } + readU16BE() { + const a = this.readU8(); + const b = this.readU8(); + return (a << 8) | b; + } + readU32LE() { + const a = this.readU8(); + const b = this.readU8(); + const c = this.readU8(); + const d = this.readU8(); + return (a | (b << 8) | (c << 16) | (d << 24)) >>> 0; + } + readU32BE() { + const a = this.readU8(); + const b = this.readU8(); + const c = this.readU8(); + const d = this.readU8(); + return ((a << 24) | (b << 16) | (c << 8) | d) >>> 0; + } + readU64LE() { + let value = 0n; + for (let shift = 0n; shift < 64n; shift += 8n) { + value |= BigInt(this.readU8()) << shift; + } + return value; + } + readU64BE() { + let value = 0n; + for (let i = 0; i < 8; i += 1) { + value = (value << 8n) | BigInt(this.readU8()); + } + return value; + } + readBytes(length) { + this.require(length); + const start = this.#offset; + this.#offset += length; + return this.bytes.subarray(start, this.#offset); + } + readRemaining() { + return this.readBytes(this.remaining); + } + skip(length) { + this.require(length); + this.#offset += length; + } + assertEnd(message = "trailing bytes") { + if (this.remaining !== 0) { + throw new LoroDecodeError(message, this.#offset); + } + } + require(length) { + if (!Number.isSafeInteger(length) || length < 0) { + throw new LoroDecodeError("invalid read length", this.#offset); + } + if (length > this.remaining) { + throw new LoroDecodeError("unexpected end of input", this.#offset); + } + } +} +export class ByteWriter { + #buffer; + #length = 0; + constructor(initialCapacity = 64) { + encodeAssert(Number.isSafeInteger(initialCapacity) && initialCapacity >= 0, "invalid writer capacity"); + this.#buffer = new Uint8Array(initialCapacity); + } + get length() { + return this.#length; + } + writeU8(value) { + assertUnsignedNumber(value, U8_MAX, "u8"); + this.ensureCapacity(1); + this.#buffer[this.#length] = value; + this.#length += 1; + } + writeU16LE(value) { + assertUnsignedNumber(value, U16_MAX, "u16"); + this.writeU8(value & U8_MAX); + this.writeU8(value >>> 8); + } + writeU16BE(value) { + assertUnsignedNumber(value, U16_MAX, "u16"); + this.writeU8(value >>> 8); + this.writeU8(value & U8_MAX); + } + writeU32LE(value) { + assertUnsignedNumber(value, U32_MAX, "u32"); + this.writeU8(value & U8_MAX); + this.writeU8((value >>> 8) & U8_MAX); + this.writeU8((value >>> 16) & U8_MAX); + this.writeU8((value >>> 24) & U8_MAX); + } + writeU32BE(value) { + assertUnsignedNumber(value, U32_MAX, "u32"); + this.writeU8((value >>> 24) & U8_MAX); + this.writeU8((value >>> 16) & U8_MAX); + this.writeU8((value >>> 8) & U8_MAX); + this.writeU8(value & U8_MAX); + } + writeU64LE(value) { + assertUnsignedBigInt(value, U64_MAX, "u64"); + for (let shift = 0n; shift < 64n; shift += 8n) { + this.writeU8(Number((value >> shift) & 0xffn)); + } + } + writeU64BE(value) { + assertUnsignedBigInt(value, U64_MAX, "u64"); + for (let shift = 56n; shift >= 0n; shift -= 8n) { + this.writeU8(Number((value >> shift) & 0xffn)); + } + } + writeBytes(bytes) { + this.ensureCapacity(bytes.length); + this.#buffer.set(bytes, this.#length); + this.#length += bytes.length; + } + toUint8Array() { + // When the buffer is exactly full, hand it over instead of copying; any + // later write would reallocate, so the returned array stays intact. This + // matches Lz4Output.finish in lz4.ts. + return this.#length === this.#buffer.length + ? this.#buffer + : this.#buffer.slice(0, this.#length); + } + ensureCapacity(extra) { + const required = this.#length + extra; + if (required <= this.#buffer.length) { + return; + } + let capacity = Math.max(64, this.#buffer.length); + while (capacity < required) { + capacity = Math.max(required, capacity * 2); + } + const next = new Uint8Array(capacity); + next.set(this.#buffer.subarray(0, this.#length)); + this.#buffer = next; + } +} +function assertUnsignedNumber(value, max, label) { + if (!Number.isSafeInteger(value) || value < 0 || value > max) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } +} +function assertUnsignedBigInt(value, max, label) { + if (value < 0n || value > max) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } +} +export function concatBytes(...parts) { + let length = 0; + for (const part of parts) { + length += part.length; + encodeAssert(Number.isSafeInteger(length), "byte sequence is too large"); + } + const output = new Uint8Array(length); + let offset = 0; + for (const part of parts) { + output.set(part, offset); + offset += part.length; + } + return output; +} +export function bytesEqual(a, b) { + if (a.length !== b.length) { + return false; + } + for (let i = 0; i < a.length; i += 1) { + if (a[i] !== b[i]) { + return false; + } + } + return true; +} +export function compareBytes(a, b) { + const length = Math.min(a.length, b.length); + for (let i = 0; i < length; i += 1) { + const diff = a[i] - b[i]; + if (diff !== 0) { + return diff; + } + } + return a.length - b.length; +} +export function bytesToHex(bytes) { + let output = ""; + for (const byte of bytes) { + output += byte.toString(16).padStart(2, "0"); + } + return output; +} +export function hexToBytes(hex) { + if (hex.length % 2 !== 0 || !/^[0-9a-f]*$/iu.test(hex)) { + throw new LoroDecodeError("invalid hexadecimal byte string"); + } + const output = new Uint8Array(hex.length / 2); + for (let i = 0; i < output.length; i += 1) { + output[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + } + return output; +} diff --git a/loro-js/src/codec/change-block-codec.js b/loro-js/src/codec/change-block-codec.js new file mode 100644 index 000000000..3afef5b2b --- /dev/null +++ b/loro-js/src/codec/change-block-codec.js @@ -0,0 +1,789 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { decodeChangeKeys, decodeChangesHeader, decodeChangesMetadata, decodeContainerArena, decodeDeleteStartIdColumns, decodeEncodedOperationColumns, encodeChangeKeys, encodeChangesHeader, encodeChangesMetadata, encodeContainerArena, encodeDeleteStartIdColumns, encodeEncodedOperationColumns, } from "./change-block-tables"; +import { decodeEncodedChangeBlock, encodeEncodedChangeBlock } from "./change-block"; +import { readChangeValueContent, encodeChangeValueContent, } from "./change-value"; +import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; +import { decodePositionArena, encodePositionArena } from "./position-arena"; +import { ContainerType, } from "./types"; +const DELETED_TREE_ROOT = { + peer: 0xffffffffffffffffn, + counter: 2147483647, +}; +function decodeChangeBlockParts(bytes) { + const encoded = decodeEncodedChangeBlock(bytes); + const header = decodeChangesHeader(encoded.header, { + changeCount: encoded.changeCount, + counterStart: encoded.counterStart, + counterLength: encoded.counterLength, + lamportStart: encoded.lamportStart, + lamportLength: encoded.lamportLength, + }); + const metadata = decodeChangesMetadata(encoded.changeMetadata, encoded.changeCount); + const keys = decodeChangeKeys(encoded.keys); + const containers = decodeContainerArena(encoded.containerIds, header.peers, keys); + const positions = decodePositionArena(encoded.positions); + const operations = decodeEncodedOperationColumns(encoded.operations); + const context = { + peers: header.peers, + keys, + positions, + deleteStartIds: decodeDeleteStartIdColumns(encoded.deleteStartIds), + deleteIndex: 0, + }; + return { encoded, header, metadata, keys, containers, positions, operations, context }; +} +export function decodeChangeBlock(bytes) { + const { encoded, header, metadata, keys, containers, positions, operations, context } = decodeChangeBlockParts(bytes); + const mutableOperations = Array.from({ length: encoded.changeCount }, () => []); + let counter = encoded.counterStart | 0; + let changeIndex = 0; + const valueReader = new ByteReader(encoded.values); + const operationId = { peer: header.peer, counter }; + for (let row = 0; row < operations.containerIndices.length; row += 1) { + const length = operations.lengths[row]; + decodeAssert(length > 0 && length <= 2147483647, "invalid operation length"); + const containerIndex = operations.containerIndices[row]; + decodeAssert(containerIndex < containers.length, "invalid operation container index"); + const container = containers[containerIndex]; + const value = readChangeValueContent(operations.valueTypes[row], valueReader); + operationId.counter = counter; + const content = decodeOperationContent(container, operations.properties[row], value, operationId, context); + mutableOperations[changeIndex].push({ + container, + counter, + length, + content, + }); + counter = checkedCounter(counter + length); + const nextBoundary = header.counters[changeIndex + 1]; + decodeAssert(nextBoundary !== undefined, "operation exceeds change boundaries"); + decodeAssert(counter <= nextBoundary, "operation crosses a change boundary"); + if (counter === nextBoundary && changeIndex + 1 < encoded.changeCount) { + changeIndex += 1; + } + } + decodeAssert(valueReader.remaining === 0, "trailing change value bytes"); + decodeAssert(context.deleteIndex === context.deleteStartIds.peerIndices.length, "unused delete start IDs"); + decodeAssert(counter === (encoded.counterStart | 0) + encoded.counterLength, "operation lengths do not fill the block counter range"); + const changes = []; + for (let index = 0; index < encoded.changeCount; index += 1) { + changes.push({ + id: { peer: header.peer, counter: header.counters[index] }, + timestamp: metadata.timestamps[index], + dependencies: header.dependencies[index], + lamport: header.lamports[index], + message: metadata.commitMessages[index], + operations: mutableOperations[index], + }); + } + return { peers: header.peers, keys, containers, positions, changes }; +} +/** Fully validates a block while retaining only its covered counter range. */ +export function validateChangeBlock(bytes) { + const { encoded, header, containers, operations, context } = decodeChangeBlockParts(bytes); + let counter = encoded.counterStart | 0; + let changeIndex = 0; + const valueReader = new ByteReader(encoded.values); + const operationId = { peer: header.peer, counter }; + for (let row = 0; row < operations.containerIndices.length; row += 1) { + const length = operations.lengths[row]; + decodeAssert(length > 0 && length <= 2147483647, "invalid operation length"); + const containerIndex = operations.containerIndices[row]; + decodeAssert(containerIndex < containers.length, "invalid operation container index"); + const value = readChangeValueContent(operations.valueTypes[row], valueReader); + operationId.counter = counter; + decodeOperationContent(containers[containerIndex], operations.properties[row], value, operationId, context, false); + counter = checkedCounter(counter + length); + const nextBoundary = header.counters[changeIndex + 1]; + decodeAssert(nextBoundary !== undefined, "operation exceeds change boundaries"); + decodeAssert(counter <= nextBoundary, "operation crosses a change boundary"); + if (counter === nextBoundary && changeIndex + 1 < encoded.changeCount) { + changeIndex += 1; + } + } + decodeAssert(valueReader.remaining === 0, "trailing change value bytes"); + decodeAssert(context.deleteIndex === context.deleteStartIds.peerIndices.length, "unused delete start IDs"); + const counterEnd = checkedCounter((encoded.counterStart | 0) + encoded.counterLength); + decodeAssert(counter === counterEnd, "operation lengths do not fill the block counter range"); + return { peer: header.peer, counterStart: encoded.counterStart | 0, counterEnd }; +} +export function encodeChangeBlock(block) { + if (block.changes.length === 0) { + throw new LoroEncodeError("cannot encode an empty change block"); + } + const firstPeer = block.changes[0].id.peer; + const tables = initializeTables(block); + if (tables.peers.length === 0) { + registerPeer(tables, firstPeer); + } + else if (tables.peers[0] !== firstPeer) { + throw new LoroEncodeError("the first peer table entry must be the block peer"); + } + const operationContainerIndices = []; + const operationProperties = []; + const operationValueTypes = []; + const operationLengths = []; + const deleteStartIds = { + peerIndices: [], + counters: [], + lengths: [], + }; + const valueWriter = new ByteWriter(); + const lengths = []; + const counters = []; + let expectedCounter = block.changes[0].id.counter; + for (const change of block.changes) { + if (change.id.peer !== firstPeer || change.id.counter !== expectedCounter) { + throw new LoroEncodeError("changes must be continuous and belong to one peer"); + } + counters.push(change.id.counter); + let atomLength = 0; + for (const operation of change.operations) { + if (operation.counter !== expectedCounter + atomLength) { + throw new LoroEncodeError("operation counters are not continuous"); + } + assertOperationLength(operation.length); + const containerIndex = registerContainer(tables, operation.container); + const encodedContent = encodeOperationContent(operation.content, tables, deleteStartIds); + const encodedValue = encodeChangeValueContent(encodedContent.value); + valueWriter.writeBytes(encodedValue.bytes); + operationContainerIndices.push(containerIndex); + operationProperties.push(encodedContent.property); + operationValueTypes.push(encodedValue.tag); + operationLengths.push(operation.length); + atomLength += operation.length; + if (atomLength > 2147483647) { + throw new LoroEncodeError("change atom length is out of range"); + } + } + if (atomLength === 0) { + throw new LoroEncodeError("changes must contain at least one operation atom"); + } + lengths.push(atomLength); + expectedCounter = checkedEncodeCounter(expectedCounter + atomLength); + } + counters.push(expectedCounter); + for (const container of tables.containers) { + if (container.kind === "root") { + registerKey(tables, container.name); + } + else { + registerPeer(tables, container.peer); + } + } + for (const change of block.changes) { + for (const dependency of change.dependencies) { + registerPeer(tables, dependency.peer); + } + } + const header = { + peer: firstPeer, + peers: tables.peers, + counters, + lengths, + lamports: block.changes.map((change) => change.lamport), + dependencies: block.changes.map((change) => change.dependencies), + }; + const counterStart = block.changes[0].id.counter; + if (counterStart < 0) { + throw new LoroEncodeError("change block counter start must be nonnegative"); + } + const counterLength = lengths.reduce((sum, length) => sum + length, 0); + const lamportStart = block.changes[0].lamport; + const finalChange = block.changes[block.changes.length - 1]; + const lamportEnd = finalChange.lamport + lengths[lengths.length - 1]; + const lamportLength = lamportEnd - lamportStart; + if (!Number.isSafeInteger(lamportStart) || + lamportStart < 0 || + lamportStart > 4294967295 || + lamportLength < 0 || + lamportLength > 4294967295) { + throw new LoroEncodeError("change block lamport range is invalid"); + } + return encodeEncodedChangeBlock({ + counterStart, + counterLength, + lamportStart, + lamportLength, + changeCount: block.changes.length, + header: encodeChangesHeader(header), + changeMetadata: encodeChangesMetadata({ + timestamps: block.changes.map((change) => change.timestamp), + commitMessages: block.changes.map((change) => change.message), + }), + containerIds: encodeContainerArena(tables.containers, tables.peers, tables.keys), + keys: encodeChangeKeys(tables.keys), + positions: encodePositionArena(tables.positions), + operations: encodeEncodedOperationColumns({ + containerIndices: operationContainerIndices, + properties: operationProperties, + valueTypes: operationValueTypes, + lengths: operationLengths, + }), + deleteStartIds: encodeDeleteStartIdColumns(deleteStartIds), + values: valueWriter.toUint8Array(), + }); +} +/** + * Encodes a block holding exactly one change, the shape documents emit per + * history record. Seeds the peer/key tables directly instead of copying the + * wrapper arrays initializeTables accepts, avoiding one block object and two + * single-element arrays per record. The bytes are identical to + * encodeChangeBlock with `{ peers: [change.id.peer], keys, containers: [], + * positions: [], changes: [change] }`; keep the shared encoding steps below in + * sync with it. + */ +export function encodeSingleChangeBlock(change, keys) { + const peer = change.id.peer; + const copiedKeys = [...keys]; + const keyIndices = new Map(); + for (let index = 0; index < copiedKeys.length; index += 1) { + // The first occurrence wins, matching initializeTables. + if (!keyIndices.has(copiedKeys[index])) { + keyIndices.set(copiedKeys[index], index); + } + } + const tables = { + peers: [peer], + peerIndices: new Map([[peer, 0]]), + keys: copiedKeys, + keyIndices, + containers: [], + rootContainerIndices: new Map(), + normalContainerIndices: new Map(), + positions: [], + positionIndices: new Map(), + }; + const operationContainerIndices = []; + const operationProperties = []; + const operationValueTypes = []; + const operationLengths = []; + const deleteStartIds = { + peerIndices: [], + counters: [], + lengths: [], + }; + const valueWriter = new ByteWriter(); + const counterStart = change.id.counter; + let atomLength = 0; + for (const operation of change.operations) { + if (operation.counter !== counterStart + atomLength) { + throw new LoroEncodeError("operation counters are not continuous"); + } + assertOperationLength(operation.length); + const containerIndex = registerContainer(tables, operation.container); + const encodedContent = encodeOperationContent(operation.content, tables, deleteStartIds); + const encodedValue = encodeChangeValueContent(encodedContent.value); + valueWriter.writeBytes(encodedValue.bytes); + operationContainerIndices.push(containerIndex); + operationProperties.push(encodedContent.property); + operationValueTypes.push(encodedValue.tag); + operationLengths.push(operation.length); + atomLength += operation.length; + if (atomLength > 2147483647) { + throw new LoroEncodeError("change atom length is out of range"); + } + } + if (atomLength === 0) { + throw new LoroEncodeError("changes must contain at least one operation atom"); + } + const counterEnd = checkedEncodeCounter(counterStart + atomLength); + for (const container of tables.containers) { + if (container.kind === "root") { + registerKey(tables, container.name); + } + else { + registerPeer(tables, container.peer); + } + } + for (const dependency of change.dependencies) { + registerPeer(tables, dependency.peer); + } + if (counterStart < 0) { + throw new LoroEncodeError("change block counter start must be nonnegative"); + } + const lamportStart = change.lamport; + if (!Number.isSafeInteger(lamportStart) || + lamportStart < 0 || + lamportStart > 4294967295) { + throw new LoroEncodeError("change block lamport range is invalid"); + } + return encodeEncodedChangeBlock({ + counterStart, + counterLength: atomLength, + lamportStart, + lamportLength: atomLength, + changeCount: 1, + header: encodeChangesHeader({ + peer, + peers: tables.peers, + counters: [counterStart, counterEnd], + lengths: [atomLength], + lamports: [lamportStart], + dependencies: [change.dependencies], + }), + changeMetadata: encodeChangesMetadata({ + timestamps: [change.timestamp], + commitMessages: [change.message], + }), + containerIds: encodeContainerArena(tables.containers, tables.peers, tables.keys), + keys: encodeChangeKeys(tables.keys), + positions: encodePositionArena(tables.positions), + operations: encodeEncodedOperationColumns({ + containerIndices: operationContainerIndices, + properties: operationProperties, + valueTypes: operationValueTypes, + lengths: operationLengths, + }), + deleteStartIds: encodeDeleteStartIdColumns(deleteStartIds), + values: valueWriter.toUint8Array(), + }); +} +function decodeOperationContent(container, property, value, operationId, context, materialize = true) { + const type = container.containerType; + if (type === ContainerType.Map) { + const key = getIndex(context.keys, property, "map key"); + if (value.type === "delete-once") { + return materialize ? { type: "map-delete", key } : undefined; + } + if (value.type === "loro-value") { + return materialize ? { type: "map-insert", key, value: value.value } : undefined; + } + throw new LoroDecodeError("invalid map operation value"); + } + if (type === ContainerType.Text) { + if (value.type === "string") { + decodeAssert(property >= 0, "invalid text insertion position"); + return materialize + ? { type: "text-insert", position: property, value: value.value } + : undefined; + } + if (value.type === "delete-sequence") { + const deletion = takeDeletion(context, materialize); + return materialize + ? { + type: "text-delete", + position: property, + length: deletion.length, + startId: deletion.startId, + } + : undefined; + } + if (value.type === "mark-start") { + decodeAssert(property >= 0, "invalid text mark position"); + const length = bigintToIndex(value.length, 4294967295, "text mark length"); + const keyIndex = bigintToIndex(value.keyIndex, context.keys.length - 1, "text mark key index"); + const end = property + length; + decodeAssert(end <= 4294967295, "text mark end is out of range"); + return materialize + ? { + type: "text-mark", + start: property, + end, + key: context.keys[keyIndex], + value: value.value, + info: value.info, + } + : undefined; + } + if (value.type === "null") { + return materialize ? { type: "text-mark-end" } : undefined; + } + throw new LoroDecodeError("invalid text operation value"); + } + if (type === ContainerType.List) { + if (value.type === "loro-value" && value.value.type === "list") { + decodeAssert(property >= 0, "invalid list insertion position"); + return materialize + ? { type: "list-insert", position: property, values: value.value.value } + : undefined; + } + if (value.type === "delete-sequence") { + const deletion = takeDeletion(context, materialize); + return materialize + ? { + type: "list-delete", + position: property, + length: deletion.length, + startId: deletion.startId, + } + : undefined; + } + throw new LoroDecodeError("invalid list operation value"); + } + if (type === ContainerType.MovableList) { + if (value.type === "loro-value" && value.value.type === "list") { + decodeAssert(property >= 0, "invalid movable-list insertion position"); + return materialize + ? { + type: "movable-list-insert", + position: property, + values: value.value.value, + } + : undefined; + } + if (value.type === "delete-sequence") { + const deletion = takeDeletion(context, materialize); + return materialize + ? { + type: "movable-list-delete", + position: property, + length: deletion.length, + startId: deletion.startId, + } + : undefined; + } + if (value.type === "list-move") { + decodeAssert(property >= 0, "invalid movable-list destination"); + const from = bigintToIndex(value.from, 4294967295, "movable-list source"); + const peer = context.peers[bigintToIndex(value.fromPeerIndex, context.peers.length - 1, "movable-list peer index")]; + const lamport = bigintToIndex(value.lamport, 4294967295, "movable-list lamport"); + return materialize + ? { + type: "movable-list-move", + from, + to: property, + elementId: { peer, lamport }, + } + : undefined; + } + if (value.type === "list-set") { + const peer = context.peers[bigintToIndex(value.peerIndex, context.peers.length - 1, "movable-list peer index")]; + return materialize + ? { + type: "movable-list-set", + elementId: { peer, lamport: value.lamport }, + value: value.value, + } + : undefined; + } + throw new LoroDecodeError("invalid movable-list operation value"); + } + if (type === ContainerType.Tree) { + if (value.type !== "raw-tree-move") { + throw new LoroDecodeError("invalid tree operation value"); + } + const subjectPeer = context.peers[bigintToIndex(value.subjectPeerIndex, context.peers.length - 1, "tree subject peer index")]; + const parentPeer = value.parentIsNull + ? undefined + : context.peers[bigintToIndex(value.parentPeerIndex, context.peers.length - 1, "tree parent peer index")]; + const deleting = parentPeer === DELETED_TREE_ROOT.peer && + value.parentCounter === DELETED_TREE_ROOT.counter; + if (deleting) { + return materialize + ? { + type: "tree-delete", + subject: { peer: subjectPeer, counter: value.subjectCounter }, + } + : undefined; + } + const position = context.positions[bigintToIndex(value.positionIndex, context.positions.length - 1, "tree position index")]; + if (!materialize) + return undefined; + const subject = { peer: subjectPeer, counter: value.subjectCounter }; + const parent = parentPeer === undefined + ? undefined + : { peer: parentPeer, counter: value.parentCounter }; + return idsEqual(subject, operationId) + ? { type: "tree-create", subject, parent, position } + : { type: "tree-move", subject, parent, position }; + } + return materialize ? { type: "future", property, value } : undefined; +} +function encodeOperationContent(content, tables, deletions) { + switch (content.type) { + case "map-insert": + return { + property: registerKey(tables, content.key), + value: { type: "loro-value", value: content.value }, + }; + case "map-delete": + return { + property: registerKey(tables, content.key), + value: { type: "delete-once" }, + }; + case "text-insert": + return { + property: assertNonnegativePosition(content.position), + value: { type: "string", value: content.value }, + }; + case "text-delete": + registerDeletion(deletions, tables, content.startId, content.length); + return { property: content.position, value: { type: "delete-sequence" } }; + case "text-mark": { + const length = content.end - content.start; + if (length < 0) { + throw new LoroEncodeError("text mark end precedes its start"); + } + return { + property: assertNonnegativePosition(content.start), + value: { + type: "mark-start", + info: content.info, + length: BigInt(length), + keyIndex: BigInt(registerKey(tables, content.key)), + value: content.value, + }, + }; + } + case "text-mark-end": + return { property: 0, value: { type: "null" } }; + case "list-insert": + return { + property: assertNonnegativePosition(content.position), + value: { type: "loro-value", value: { type: "list", value: content.values } }, + }; + case "list-delete": + registerDeletion(deletions, tables, content.startId, content.length); + return { property: content.position, value: { type: "delete-sequence" } }; + case "movable-list-insert": + return { + property: assertNonnegativePosition(content.position), + value: { type: "loro-value", value: { type: "list", value: content.values } }, + }; + case "movable-list-delete": + registerDeletion(deletions, tables, content.startId, content.length); + return { property: content.position, value: { type: "delete-sequence" } }; + case "movable-list-move": + return { + property: assertNonnegativePosition(content.to), + value: { + type: "list-move", + from: BigInt(assertNonnegativePosition(content.from)), + fromPeerIndex: BigInt(registerPeer(tables, content.elementId.peer)), + lamport: BigInt(content.elementId.lamport), + }, + }; + case "movable-list-set": + return { + property: 0, + value: { + type: "list-set", + peerIndex: BigInt(registerPeer(tables, content.elementId.peer)), + lamport: content.elementId.lamport, + value: content.value, + }, + }; + case "tree-create": + case "tree-move": + return encodeTreeMove(content.subject, content.parent, content.position, tables); + case "tree-delete": + return encodeTreeMove(content.subject, DELETED_TREE_ROOT, new Uint8Array(), tables, true); + case "future": + return { property: content.property, value: content.value }; + } +} +function encodeTreeMove(subject, parent, position, tables, deleting = false) { + return { + property: 0, + value: { + type: "raw-tree-move", + subjectPeerIndex: BigInt(registerPeer(tables, subject.peer)), + subjectCounter: subject.counter, + positionIndex: deleting ? 0n : BigInt(registerPosition(tables, position)), + parentIsNull: parent === undefined, + parentPeerIndex: parent === undefined ? 0n : BigInt(registerPeer(tables, parent.peer)), + parentCounter: parent?.counter ?? 0, + }, + }; +} +function takeDeletion(context, materialize) { + const peerIndexValue = context.deleteStartIds.peerIndices[context.deleteIndex]; + const counter = context.deleteStartIds.counters[context.deleteIndex]; + const length = context.deleteStartIds.lengths[context.deleteIndex]; + decodeAssert(peerIndexValue !== undefined && counter !== undefined && length !== undefined, "delete start ID underflow"); + context.deleteIndex += 1; + const peerIndex = bigintToIndex(peerIndexValue, context.peers.length - 1, "delete start peer index"); + return materialize + ? { + startId: { peer: context.peers[peerIndex], counter }, + length, + } + : undefined; +} +function registerDeletion(deletions, tables, startId, length) { + deletions.peerIndices.push(BigInt(registerPeer(tables, startId.peer))); + deletions.counters.push(startId.counter); + deletions.lengths.push(length); +} +function initializeTables(block) { + const peers = [...block.peers]; + const keys = [...block.keys]; + const peerIndices = new Map(); + const keyIndices = new Map(); + for (let index = 0; index < peers.length; index += 1) { + if (peerIndices.has(peers[index])) { + throw new LoroEncodeError("duplicate peer table entry"); + } + peerIndices.set(peers[index], index); + } + for (let index = 0; index < keys.length; index += 1) { + if (!keyIndices.has(keys[index])) { + keyIndices.set(keys[index], index); + } + } + const tables = { + peers, + peerIndices, + keys, + keyIndices, + containers: block.containers.map(cloneContainerId), + rootContainerIndices: new Map(), + normalContainerIndices: new Map(), + positions: block.positions.map((position) => position.slice()), + positionIndices: new Map(), + }; + // The first occurrence wins, matching the former linear-scan lookups. + for (let index = 0; index < tables.containers.length; index += 1) { + const container = tables.containers[index]; + if (lookupContainerIndex(tables, container) === undefined) { + recordContainerIndex(tables, container, index); + } + } + for (let index = 0; index < tables.positions.length; index += 1) { + const key = positionKey(tables.positions[index]); + if (!tables.positionIndices.has(key)) { + tables.positionIndices.set(key, index); + } + } + return tables; +} +function registerPeer(tables, peer) { + const current = tables.peerIndices.get(peer); + if (current !== undefined) { + return current; + } + if (peer < 0n || peer > 0xffffffffffffffffn) { + throw new LoroEncodeError(`peer ID is out of range: ${peer}`); + } + const index = tables.peers.length; + tables.peers.push(peer); + tables.peerIndices.set(peer, index); + return index; +} +function registerKey(tables, key) { + const current = tables.keyIndices.get(key); + if (current !== undefined) { + return current; + } + const index = tables.keys.length; + tables.keys.push(key); + tables.keyIndices.set(key, index); + return index; +} +function registerContainer(tables, container) { + const current = lookupContainerIndex(tables, container); + if (current !== undefined) { + return current; + } + tables.containers.push(cloneContainerId(container)); + const index = tables.containers.length - 1; + recordContainerIndex(tables, tables.containers[index], index); + return index; +} +function registerPosition(tables, position) { + const key = positionKey(position); + const current = tables.positionIndices.get(key); + if (current !== undefined) { + return current; + } + tables.positions.push(position.slice()); + const index = tables.positions.length - 1; + tables.positionIndices.set(key, index); + return index; +} +function containerTypeKey(type) { + return typeof type === "string" ? type : `#${type.value}`; +} +function lookupContainerIndex(tables, container) { + if (container.kind === "root") { + return tables.rootContainerIndices + .get(container.name) + ?.get(containerTypeKey(container.containerType)); + } + return tables.normalContainerIndices + .get(container.peer) + ?.get(container.counter) + ?.get(containerTypeKey(container.containerType)); +} +function recordContainerIndex(tables, container, index) { + const typeKey = containerTypeKey(container.containerType); + if (container.kind === "root") { + let byType = tables.rootContainerIndices.get(container.name); + if (byType === undefined) { + byType = new Map(); + tables.rootContainerIndices.set(container.name, byType); + } + byType.set(typeKey, index); + return; + } + let byCounter = tables.normalContainerIndices.get(container.peer); + if (byCounter === undefined) { + byCounter = new Map(); + tables.normalContainerIndices.set(container.peer, byCounter); + } + let byType = byCounter.get(container.counter); + if (byType === undefined) { + byType = new Map(); + byCounter.set(container.counter, byType); + } + byType.set(typeKey, index); +} +// Maps position bytes one-to-one to code units so distinct byte sequences +// never share a key. +function positionKey(position) { + let key = ""; + for (let index = 0; index < position.length; index += 1) { + key += String.fromCharCode(position[index]); + } + return key; +} +function getIndex(values, index, label) { + decodeAssert(Number.isSafeInteger(index) && index >= 0 && index < values.length, `invalid ${label} index`); + return values[index]; +} +const MAX_SAFE_BIGINT = 0x1fffffffffffffn; +function bigintToIndex(value, max, label) { + decodeAssert(value >= 0n && value <= MAX_SAFE_BIGINT, `${label} is out of range`); + const result = Number(value); + decodeAssert(max >= 0 && result <= max, `${label} is out of range`); + return result; +} +function assertNonnegativePosition(value) { + if (!Number.isSafeInteger(value) || value < 0 || value > 2147483647) { + throw new LoroEncodeError(`position is out of range: ${value}`); + } + return value; +} +function assertOperationLength(value) { + if (!Number.isSafeInteger(value) || value <= 0 || value > 2147483647) { + throw new LoroEncodeError(`operation length is out of range: ${value}`); + } +} +function checkedCounter(value) { + decodeAssert(Number.isSafeInteger(value) && value >= -2147483648 && value <= 2147483647, "operation counter overflow"); + return value; +} +function checkedEncodeCounter(value) { + if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { + throw new LoroEncodeError(`operation counter is out of range: ${value}`); + } + return value; +} +function idsEqual(left, right) { + return left.peer === right.peer && left.counter === right.counter; +} +function cloneContainerId(container) { + return container.kind === "root" + ? { + kind: "root", + name: container.name, + containerType: container.containerType, + } + : { + kind: "normal", + peer: container.peer, + counter: container.counter, + containerType: container.containerType, + }; +} diff --git a/loro-js/src/codec/change-block-tables.js b/loro-js/src/codec/change-block-tables.js new file mode 100644 index 000000000..b7a557973 --- /dev/null +++ b/loro-js/src/codec/change-block-tables.js @@ -0,0 +1,375 @@ +import { ByteReader, ByteWriter, concatBytes } from "./bytes"; +import { containerTypeFromRawByte, containerTypeToRawByte } from "./container-id"; +import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; +import { readUlebNumber, writeUleb128 } from "./leb128"; +import { PostcardReader, PostcardWriter } from "./postcard"; +import { decodeColumnarVecMaybeWrapped, decodeDeltaRleI32, decodeDeltaRleIsize, decodeDeltaRleU32, decodeDeltaRleUsize, decodeRleU8, decodeRleU32, encodeAnyRleU32, encodeAnyRleUsize, encodeBoolRle, encodeColumnarVecWrapped, encodeDeltaOfDeltaI64, encodeDeltaRleI32, encodeDeltaRleIsize, encodeDeltaRleU32, encodeDeltaRleUsize, encodeRleU8, encodeRleU32, takeAnyRleU32, takeAnyRleUsize, takeBoolRle, takeDeltaOfDeltaI64, } from "./serde-columnar"; +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const textEncoder = new TextEncoder(); +const I32_MIN = -2147483648; +const I32_MAX = 2147483647; +export function decodeChangesHeader(bytes, options) { + const changeCount = options.changeCount; + decodeAssert(Number.isSafeInteger(changeCount) && changeCount > 0 && changeCount <= 10000000, "invalid change count"); + const firstCounter = options.counterStart | 0; + decodeAssert(options.counterLength <= I32_MAX, "change counter length is out of range"); + const reader = new ByteReader(bytes); + const peerCount = readUlebNumber(reader, 10000000); + decodeAssert(peerCount > 0, "change header has an empty peer table"); + const peers = []; + for (let index = 0; index < peerCount; index += 1) { + peers.push(reader.readU64LE()); + } + const lengths = []; + let knownLength = 0; + for (let index = 0; index < changeCount - 1; index += 1) { + const length = readUlebNumber(reader, I32_MAX); + knownLength = checkedI32(knownLength + length, "change counter length"); + lengths.push(length); + } + const finalLength = options.counterLength - knownLength; + decodeAssert(finalLength >= 0, "change lengths exceed the block counter range"); + lengths.push(finalLength); + let remaining = bytes.subarray(reader.position); + let dependencyOnSelf; + [dependencyOnSelf, remaining] = takeBoolRle(remaining, changeCount); + let dependencyLengthsBigInt; + [dependencyLengthsBigInt, remaining] = takeAnyRleUsize(remaining, changeCount); + const dependencyLengths = dependencyLengthsBigInt.map((value) => bigintToNumber(value, 10000000, "dependency length")); + const otherDependencyCount = dependencyLengths.reduce((sum, value) => sum + value, 0); + decodeAssert(otherDependencyCount <= 10000000, "too many change dependencies"); + let dependencyPeerIndices; + [dependencyPeerIndices, remaining] = takeAnyRleUsize(remaining, otherDependencyCount); + let dependencyCounters; + [dependencyCounters, remaining] = takeDeltaOfDeltaI64(remaining, otherDependencyCount); + const dependencies = []; + let dependencyIndex = 0; + let counter = firstCounter; + for (let index = 0; index < changeCount; index += 1) { + const ids = []; + if (dependencyOnSelf[index]) { + decodeAssert(counter > I32_MIN, "self dependency counter underflow"); + ids.push({ peer: peers[0], counter: counter - 1 }); + } + for (let dep = 0; dep < dependencyLengths[index]; dep += 1) { + const peerIndex = bigintToNumber(dependencyPeerIndices[dependencyIndex], peers.length - 1, "dependency peer index"); + const dependencyCounter = bigintToNumber(dependencyCounters[dependencyIndex], I32_MAX, "dependency counter"); + ids.push({ peer: peers[peerIndex], counter: dependencyCounter }); + dependencyIndex += 1; + } + dependencies.push(ids); + counter = checkedI32(counter + lengths[index], "change counter"); + } + decodeAssert(dependencyIndex === dependencyPeerIndices.length && + dependencyIndex === dependencyCounters.length, "trailing change dependencies"); + const counters = []; + counter = firstCounter; + for (const length of lengths) { + counters.push(counter); + counter = checkedI32(counter + length, "change counter"); + } + counters.push(counter); + let encodedLamports; + [encodedLamports, remaining] = takeDeltaOfDeltaI64(remaining, changeCount - 1); + decodeAssert(remaining.length === 0, "trailing change header bytes"); + const lamports = encodedLamports.map((value) => bigintToNumber(value, 4294967295, "lamport")); + const blockLamportEnd = options.lamportStart + options.lamportLength; + decodeAssert(blockLamportEnd <= 4294967295, "lamport range overflow"); + const lastLamport = blockLamportEnd - finalLength; + decodeAssert(Number.isSafeInteger(lastLamport) && lastLamport >= 0 && lastLamport <= 4294967295, "invalid final lamport"); + lamports.push(lastLamport); + return { + peer: peers[0], + peers, + counters, + lengths, + lamports, + dependencies, + }; +} +export function encodeChangesHeader(header) { + const changeCount = header.lengths.length; + if (changeCount === 0 || + header.peers.length === 0 || + header.peers[0] !== header.peer || + header.dependencies.length !== changeCount || + header.lamports.length !== changeCount) { + throw new LoroEncodeError("inconsistent change header arrays"); + } + const writer = new ByteWriter(); + writeUleb128(writer, header.peers.length); + for (const peer of header.peers) { + writer.writeU64LE(peer); + } + for (let index = 0; index < changeCount - 1; index += 1) { + writeUleb128(writer, assertNonnegativeI32(header.lengths[index], "change length")); + } + const peerIndices = new Map(header.peers.map((peer, index) => [peer, index])); + const selfDependencies = []; + const dependencyLengths = []; + const dependencyPeerIndices = []; + const dependencyCounters = []; + for (let index = 0; index < changeCount; index += 1) { + const expectedSelfCounter = header.counters[index] - 1; + let hasSelf = false; + let otherCount = 0; + for (const dependency of header.dependencies[index]) { + if (dependency.peer === header.peer) { + if (hasSelf || dependency.counter !== expectedSelfCounter) { + throw new LoroEncodeError("invalid same-peer change dependency"); + } + hasSelf = true; + } + else { + const peerIndex = peerIndices.get(dependency.peer); + if (peerIndex === undefined) { + throw new LoroEncodeError(`dependency peer is absent from the peer table`); + } + dependencyPeerIndices.push(BigInt(peerIndex)); + dependencyCounters.push(BigInt(assertNonnegativeI32(dependency.counter, "dependency counter"))); + otherCount += 1; + } + } + selfDependencies.push(hasSelf); + dependencyLengths.push(BigInt(otherCount)); + } + writer.writeBytes(encodeBoolRle(selfDependencies)); + writer.writeBytes(encodeAnyRleUsize(dependencyLengths)); + writer.writeBytes(encodeAnyRleUsize(dependencyPeerIndices)); + writer.writeBytes(encodeDeltaOfDeltaI64(dependencyCounters)); + writer.writeBytes(encodeDeltaOfDeltaI64(header.lamports.slice(0, -1).map((value) => BigInt(value)))); + return writer.toUint8Array(); +} +export function decodeChangesMetadata(bytes, changeCount) { + let remaining = bytes; + let timestamps; + [timestamps, remaining] = takeDeltaOfDeltaI64(remaining, changeCount); + let messageLengths; + [messageLengths, remaining] = takeAnyRleU32(remaining, changeCount); + const totalLength = messageLengths.reduce((sum, value) => sum + value, 0); + decodeAssert(totalLength === remaining.length, "commit message byte length mismatch"); + const commitMessages = []; + let offset = 0; + for (const length of messageLengths) { + if (length === 0) { + commitMessages.push(undefined); + continue; + } + const messageBytes = remaining.subarray(offset, offset + length); + try { + commitMessages.push(textDecoder.decode(messageBytes)); + } + catch { + throw new LoroDecodeError("invalid UTF-8 commit message", offset); + } + offset += length; + } + return { timestamps, commitMessages }; +} +export function encodeChangesMetadata(metadata) { + if (metadata.timestamps.length !== metadata.commitMessages.length) { + throw new LoroEncodeError("inconsistent change metadata arrays"); + } + const lengths = []; + const messages = []; + for (const message of metadata.commitMessages) { + if (message === undefined) { + lengths.push(0); + } + else { + const bytes = textEncoder.encode(message); + lengths.push(bytes.length); + messages.push(bytes); + } + } + return concatBytes(encodeDeltaOfDeltaI64(metadata.timestamps), encodeAnyRleU32(lengths), ...messages); +} +export function decodeChangeKeys(bytes) { + const reader = new ByteReader(bytes); + const keys = []; + while (reader.remaining > 0) { + const offset = reader.position; + const value = reader.readBytes(readUlebNumber(reader, 2147483647)); + try { + keys.push(textDecoder.decode(value)); + } + catch { + throw new LoroDecodeError("invalid UTF-8 change key", offset); + } + } + return keys; +} +export function encodeChangeKeys(keys) { + const writer = new ByteWriter(); + for (const key of keys) { + const bytes = textEncoder.encode(key); + writeUleb128(writer, bytes.length); + writer.writeBytes(bytes); + } + return writer.toUint8Array(); +} +export function decodeContainerArena(bytes, peers, keys) { + const reader = new PostcardReader(bytes); + const count = reader.readUsize(); + decodeAssert(count <= 10000000, "container arena is too large"); + const containers = []; + for (let index = 0; index < count; index += 1) { + decodeAssert(reader.readUsize() === 4, "invalid encoded container field count"); + const isRoot = reader.readBool(); + const containerType = containerTypeFromRawByte(reader.readU8()); + const peerIndex = reader.readUsize(); + const keyIndexOrCounter = reader.readI32(); + if (isRoot) { + decodeAssert(keyIndexOrCounter >= 0 && keyIndexOrCounter < keys.length, "invalid root container key index"); + containers.push({ + kind: "root", + name: keys[keyIndexOrCounter], + containerType, + }); + } + else { + decodeAssert(peerIndex < peers.length, "invalid normal container peer index"); + containers.push({ + kind: "normal", + peer: peers[peerIndex], + counter: keyIndexOrCounter, + containerType, + }); + } + } + reader.assertEnd(); + return containers; +} +export function encodeContainerArena(containers, peers, keys) { + const writer = new PostcardWriter(); + writer.writeUsize(containers.length); + const peerIndices = new Map(peers.map((peer, index) => [peer, index])); + const keyIndices = new Map(keys.map((key, index) => [key, index])); + for (const container of containers) { + writer.writeUsize(4); + writer.writeBool(container.kind === "root"); + writer.writeU8(containerTypeToRawByte(container.containerType)); + if (container.kind === "root") { + const keyIndex = keyIndices.get(container.name); + if (keyIndex === undefined) { + throw new LoroEncodeError(`root container name is absent from the key table`); + } + writer.writeUsize(0); + writer.writeI32(keyIndex); + } + else { + const peerIndex = peerIndices.get(container.peer); + if (peerIndex === undefined) { + throw new LoroEncodeError(`container peer is absent from the peer table`); + } + writer.writeUsize(peerIndex); + writer.writeI32(container.counter); + } + } + return writer.toUint8Array(); +} +export function decodeEncodedOperations(bytes) { + const decoded = decodeEncodedOperationColumns(bytes); + return decoded.containerIndices.map((containerIndex, index) => ({ + containerIndex, + property: decoded.properties[index], + valueType: decoded.valueTypes[index], + length: decoded.lengths[index], + })); +} +export function decodeEncodedOperationColumns(bytes) { + if (bytes.length === 0) { + return { containerIndices: [], properties: [], valueTypes: [], lengths: [] }; + } + const columns = decodeColumnarVecMaybeWrapped(bytes); + decodeAssert(columns.length === 4, "encoded operations must have four columns"); + const containerIndices = decodeDeltaRleU32(columns[0]); + const properties = decodeDeltaRleI32(columns[1]); + const valueTypes = decodeRleU8(columns[2]); + const lengths = decodeRleU32(columns[3]); + decodeAssert(properties.length === containerIndices.length && + valueTypes.length === containerIndices.length && + lengths.length === containerIndices.length, "encoded operation column length mismatch"); + return { containerIndices, properties, valueTypes, lengths }; +} +export function encodeEncodedOperations(rows) { + return encodeEncodedOperationColumns({ + containerIndices: rows.map((row) => row.containerIndex), + properties: rows.map((row) => row.property), + valueTypes: rows.map((row) => row.valueType), + lengths: rows.map((row) => row.length), + }); +} +/** Encodes operation columns directly, without intermediate row objects. */ +export function encodeEncodedOperationColumns(columns) { + if (columns.properties.length !== columns.containerIndices.length || + columns.valueTypes.length !== columns.containerIndices.length || + columns.lengths.length !== columns.containerIndices.length) { + throw new LoroEncodeError("inconsistent encoded operation columns"); + } + return encodeColumnarVecWrapped([ + encodeDeltaRleU32(columns.containerIndices), + encodeDeltaRleI32(columns.properties), + encodeRleU8(columns.valueTypes), + encodeRleU32(columns.lengths), + ]); +} +export function decodeDeleteStartIds(bytes) { + const decoded = decodeDeleteStartIdColumns(bytes); + return decoded.peerIndices.map((peerIndex, index) => ({ + peerIndex, + counter: decoded.counters[index], + length: decoded.lengths[index], + })); +} +export function decodeDeleteStartIdColumns(bytes) { + if (bytes.length === 0) { + return { peerIndices: [], counters: [], lengths: [] }; + } + const columns = decodeColumnarVecMaybeWrapped(bytes); + decodeAssert(columns.length === 3, "delete start IDs must have three columns"); + const peerIndices = decodeDeltaRleUsize(columns[0]); + const counters = decodeDeltaRleI32(columns[1]); + const lengths = decodeDeltaRleIsize(columns[2]); + decodeAssert(counters.length === peerIndices.length && lengths.length === peerIndices.length, "delete start ID column length mismatch"); + return { peerIndices, counters, lengths }; +} +export function encodeDeleteStartIds(rows) { + return encodeDeleteStartIdColumns({ + peerIndices: rows.map((row) => row.peerIndex), + counters: rows.map((row) => row.counter), + lengths: rows.map((row) => row.length), + }); +} +/** Encodes delete start ID columns directly, without intermediate row objects. */ +export function encodeDeleteStartIdColumns(columns) { + if (columns.counters.length !== columns.peerIndices.length || + columns.lengths.length !== columns.peerIndices.length) { + throw new LoroEncodeError("inconsistent delete start ID columns"); + } + if (columns.peerIndices.length === 0) { + return new Uint8Array(); + } + return encodeColumnarVecWrapped([ + encodeDeltaRleUsize(columns.peerIndices), + encodeDeltaRleI32(columns.counters), + encodeDeltaRleIsize(columns.lengths), + ]); +} +function checkedI32(value, label) { + decodeAssert(Number.isSafeInteger(value) && value >= I32_MIN && value <= I32_MAX, `${label} overflow`); + return value; +} +const MAX_SAFE_BIGINT = 0x1fffffffffffffn; +function bigintToNumber(value, max, label) { + decodeAssert(value >= 0n && value <= MAX_SAFE_BIGINT, `${label} is out of range`); + const result = Number(value); + decodeAssert(result <= max, `${label} is out of range`); + return result; +} +function assertNonnegativeI32(value, label) { + if (!Number.isSafeInteger(value) || value < 0 || value > I32_MAX) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } + return value; +} diff --git a/loro-js/src/codec/change-block.js b/loro-js/src/codec/change-block.js new file mode 100644 index 000000000..e7ddc2658 --- /dev/null +++ b/loro-js/src/codec/change-block.js @@ -0,0 +1,38 @@ +import { PostcardReader, PostcardWriter } from "./postcard"; +export function decodeEncodedChangeBlock(bytes) { + const reader = new PostcardReader(bytes); + const block = { + counterStart: reader.readU32(), + counterLength: reader.readU32(), + lamportStart: reader.readU32(), + lamportLength: reader.readU32(), + changeCount: reader.readU32(), + header: reader.readBytes(), + changeMetadata: reader.readBytes(), + containerIds: reader.readBytes(), + keys: reader.readBytes(), + positions: reader.readBytes(), + operations: reader.readBytes(), + deleteStartIds: reader.readBytes(), + values: reader.readBytes(), + }; + reader.assertEnd(); + return block; +} +export function encodeEncodedChangeBlock(block) { + const writer = new PostcardWriter(); + writer.writeU32(block.counterStart); + writer.writeU32(block.counterLength); + writer.writeU32(block.lamportStart); + writer.writeU32(block.lamportLength); + writer.writeU32(block.changeCount); + writer.writeBytes(block.header); + writer.writeBytes(block.changeMetadata); + writer.writeBytes(block.containerIds); + writer.writeBytes(block.keys); + writer.writeBytes(block.positions); + writer.writeBytes(block.operations); + writer.writeBytes(block.deleteStartIds); + writer.writeBytes(block.values); + return writer.toUint8Array(); +} diff --git a/loro-js/src/codec/change-value.js b/loro-js/src/codec/change-value.js new file mode 100644 index 000000000..58c2c537f --- /dev/null +++ b/loro-js/src/codec/change-value.js @@ -0,0 +1,375 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; +import { readSleb128, readUleb128, writeSleb128, writeUleb128 } from "./leb128"; +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const textEncoder = new TextEncoder(); +const MAX_VALUE_DEPTH = 1024; +const MAX_COLLECTION_LENGTH = 10000000; +const MAX_COLLECTION_LENGTH_BIGINT = BigInt(MAX_COLLECTION_LENGTH); +const I32_MIN_BIGINT = -0x80000000n; +const I32_MAX_BIGINT = 0x7fffffffn; +const U32_MAX_BIGINT = 0xffffffffn; +// Shared scratch for big-endian f64 access; never exposed to callers. +const f64Scratch = new Uint8Array(8); +const f64View = new DataView(f64Scratch.buffer); +export function decodeChangeValue(bytes) { + const reader = new ByteReader(bytes); + const tag = reader.readU8(); + const value = readChangeValueContent(tag, reader); + decodeAssert(reader.remaining === 0, "trailing change value bytes"); + return value; +} +export function decodeChangeValueContent(tag, bytes) { + const reader = new ByteReader(bytes); + const value = readChangeValueContent(tag, reader); + return [value, bytes.subarray(reader.position)]; +} +/** + * Reads one tagged change value payload from the reader, leaving it positioned + * after the value. Per-operation decode loops share one reader across values to + * avoid the tuple and subarray allocations of decodeChangeValueContent. + */ +export function readChangeValueContent(tag, reader) { + const kind = tag & 0x7f; + switch (kind) { + case 0: + return { type: "null" }; + case 1: + return { type: "bool", value: true }; + case 2: + return { type: "bool", value: false }; + case 3: + return { type: "i64", value: readSleb128(reader) }; + case 4: + return { type: "double", value: readF64BE(reader) }; + case 5: + return { type: "string", value: readUtf8(reader) }; + case 6: + return { type: "binary", value: readLengthPrefixedBytes(reader) }; + case 7: + return { type: "container-index", value: readUleb128(reader) }; + case 8: + return { type: "delete-once" }; + case 9: + return { type: "delete-sequence" }; + case 10: { + const integer = readSleb128(reader); + decodeAssert(integer >= I32_MIN_BIGINT && integer <= I32_MAX_BIGINT, "change delta integer is out of range"); + return { type: "delta-int", value: Number(integer) }; + } + case 11: + return { type: "loro-value", value: readChangeLoroValue(reader) }; + case 12: + return { + type: "mark-start", + info: reader.readU8(), + length: readUleb128(reader), + keyIndex: readUleb128(reader), + value: readChangeLoroValue(reader), + }; + case 13: { + const targetIndex = readUleb128(reader); + const parentIsNull = reader.readU8() !== 0; + const position = readUleb128(reader); + return { + type: "tree-move", + targetIndex, + parentIsNull, + position, + parentIndex: parentIsNull ? undefined : readUleb128(reader), + }; + } + case 14: + return { + type: "list-move", + from: readUleb128(reader), + fromPeerIndex: readUleb128(reader), + lamport: readUleb128(reader), + }; + case 15: { + const peerIndex = readUleb128(reader); + const lamport = readUleb128(reader); + decodeAssert(lamport <= U32_MAX_BIGINT, "list-set lamport is out of range"); + return { + type: "list-set", + peerIndex, + lamport: Number(lamport), + value: readChangeLoroValue(reader), + }; + } + case 16: { + const subjectPeerIndex = readUleb128(reader); + const subjectCounter = readNonnegativeI32(reader, "tree subject counter"); + const positionIndex = readUleb128(reader); + const parentIsNull = reader.readU8() !== 0; + const parentPeerIndex = parentIsNull ? 0n : readUleb128(reader); + const parentCounter = parentIsNull + ? 0 + : readNonnegativeI32(reader, "tree parent counter"); + return { + type: "raw-tree-move", + subjectPeerIndex, + subjectCounter, + positionIndex, + parentIsNull, + parentPeerIndex, + parentCounter, + }; + } + default: + return { type: "future", tag, data: readLengthPrefixedBytes(reader) }; + } +} +export function encodeChangeValue(value) { + const encoded = encodeChangeValueContent(value); + const writer = new ByteWriter(encoded.bytes.length + 1); + writer.writeU8(encoded.tag); + writer.writeBytes(encoded.bytes); + return writer.toUint8Array(); +} +export function encodeChangeValueContent(value) { + const writer = new ByteWriter(); + let tag; + switch (value.type) { + case "null": + tag = 0; + break; + case "bool": + tag = value.value ? 1 : 2; + break; + case "i64": + tag = 3; + writeSleb128(writer, value.value); + break; + case "double": + tag = 4; + writeF64BE(writer, value.value); + break; + case "string": + tag = 5; + writeUtf8(writer, value.value); + break; + case "binary": + tag = 6; + writeLengthPrefixedBytes(writer, value.value); + break; + case "container-index": + tag = 7; + writeUleb128(writer, value.value); + break; + case "delete-once": + tag = 8; + break; + case "delete-sequence": + tag = 9; + break; + case "delta-int": + tag = 10; + assertI32(value.value, "delta integer"); + writeSleb128(writer, value.value); + break; + case "loro-value": + tag = 11; + writeChangeLoroValue(writer, value.value); + break; + case "mark-start": + tag = 12; + writer.writeU8(value.info); + writeUleb128(writer, value.length); + writeUleb128(writer, value.keyIndex); + writeChangeLoroValue(writer, value.value); + break; + case "tree-move": + tag = 13; + writeUleb128(writer, value.targetIndex); + writer.writeU8(value.parentIsNull ? 1 : 0); + writeUleb128(writer, value.position); + if (!value.parentIsNull) { + if (value.parentIndex === undefined) { + throw new LoroEncodeError("tree move lacks its parent index"); + } + writeUleb128(writer, value.parentIndex); + } + break; + case "list-move": + tag = 14; + writeUleb128(writer, value.from); + writeUleb128(writer, value.fromPeerIndex); + writeUleb128(writer, value.lamport); + break; + case "list-set": + tag = 15; + writeUleb128(writer, value.peerIndex); + writeUleb128(writer, value.lamport); + writeChangeLoroValue(writer, value.value); + break; + case "raw-tree-move": + tag = 16; + writeUleb128(writer, value.subjectPeerIndex); + writeUleb128(writer, assertNonnegativeI32(value.subjectCounter, "tree subject counter")); + writeUleb128(writer, value.positionIndex); + writer.writeU8(value.parentIsNull ? 1 : 0); + if (!value.parentIsNull) { + writeUleb128(writer, value.parentPeerIndex); + writeUleb128(writer, assertNonnegativeI32(value.parentCounter, "tree parent counter")); + } + break; + case "future": + tag = value.tag; + if (!Number.isSafeInteger(tag) || tag < 0 || tag > 0xff || (tag & 0x7f) <= 16) { + throw new LoroEncodeError(`invalid future change value tag: ${tag}`); + } + writeLengthPrefixedBytes(writer, value.data); + } + return { tag, bytes: writer.toUint8Array() }; +} +export function readChangeLoroValue(reader, depth = 0) { + decodeAssert(depth <= MAX_VALUE_DEPTH, "change LoroValue is too deep", reader.position); + const kind = reader.readU8(); + switch (kind) { + case 0: + return { type: "null" }; + case 1: + return { type: "bool", value: true }; + case 2: + return { type: "bool", value: false }; + case 3: + return { type: "i64", value: readSleb128(reader) }; + case 4: + return { type: "double", value: readF64BE(reader) }; + case 5: + return { type: "string", value: readUtf8(reader) }; + case 6: + return { type: "binary", value: readLengthPrefixedBytes(reader) }; + case 7: { + const length = readCollectionLength(reader); + decodeAssert(length <= reader.remaining, "change LoroValue list length exceeds input"); + const value = []; + for (let index = 0; index < length; index += 1) { + value.push(readChangeLoroValue(reader, depth + 1)); + } + return { type: "list", value }; + } + case 8: { + const length = readCollectionLength(reader); + decodeAssert(length <= Math.floor(reader.remaining / 2), "change LoroValue map length exceeds input"); + const value = []; + for (let index = 0; index < length; index += 1) { + value.push([readUleb128(reader), readChangeLoroValue(reader, depth + 1)]); + } + return { type: "map", value }; + } + case 9: + return { type: "container-type", value: reader.readU8() }; + default: + throw new LoroDecodeError("invalid change LoroValue kind", reader.position - 1); + } +} +export function writeChangeLoroValue(writer, value, depth = 0) { + if (depth > MAX_VALUE_DEPTH) { + throw new LoroEncodeError("change LoroValue is too deep"); + } + switch (value.type) { + case "null": + writer.writeU8(0); + return; + case "bool": + writer.writeU8(value.value ? 1 : 2); + return; + case "i64": + writer.writeU8(3); + writeSleb128(writer, value.value); + return; + case "double": + writer.writeU8(4); + writeF64BE(writer, value.value); + return; + case "string": + writer.writeU8(5); + writeUtf8(writer, value.value); + return; + case "binary": + writer.writeU8(6); + writeLengthPrefixedBytes(writer, value.value); + return; + case "list": + writer.writeU8(7); + assertCollectionLength(value.value.length); + writeUleb128(writer, value.value.length); + for (const item of value.value) { + writeChangeLoroValue(writer, item, depth + 1); + } + return; + case "map": + writer.writeU8(8); + assertCollectionLength(value.value.length); + writeUleb128(writer, value.value.length); + for (const [keyIndex, item] of value.value) { + writeUleb128(writer, keyIndex); + writeChangeLoroValue(writer, item, depth + 1); + } + return; + case "container-type": + writer.writeU8(9); + writer.writeU8(value.value); + } +} +function readCollectionLength(reader) { + const value = readUleb128(reader, MAX_COLLECTION_LENGTH_BIGINT); + return Number(value); +} +function readNonnegativeI32(reader, label) { + const value = readUleb128(reader, I32_MAX_BIGINT); + decodeAssert(value <= I32_MAX_BIGINT, `${label} is out of range`); + return Number(value); +} +function readUtf8(reader) { + const offset = reader.position; + const bytes = readLengthPrefixedBytes(reader); + try { + return textDecoder.decode(bytes); + } + catch { + throw new LoroDecodeError("invalid UTF-8 change value", offset); + } +} +function writeUtf8(writer, value) { + writeLengthPrefixedBytes(writer, textEncoder.encode(value)); +} +function readLengthPrefixedBytes(reader) { + const length = readUleb128(reader, I32_MAX_BIGINT); + return reader.readBytes(Number(length)); +} +function writeLengthPrefixedBytes(writer, value) { + writeUleb128(writer, value.length); + writer.writeBytes(value); +} +function readF64BE(reader) { + const offset = reader.position; + reader.skip(8); + const bytes = reader.bytes; + for (let index = 0; index < 8; index += 1) { + f64Scratch[index] = bytes[offset + index]; + } + return f64View.getFloat64(0, false); +} +function writeF64BE(writer, value) { + f64View.setFloat64(0, value, false); + writer.writeBytes(f64Scratch); +} +function assertI32(value, label) { + if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } +} +function assertNonnegativeI32(value, label) { + if (!Number.isSafeInteger(value) || value < 0 || value > 2147483647) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } + return value; +} +function assertCollectionLength(length) { + if (!Number.isSafeInteger(length) || length < 0 || length > MAX_COLLECTION_LENGTH) { + throw new LoroEncodeError(`change value collection is too large: ${length}`); + } +} diff --git a/loro-js/src/codec/container-id.js b/loro-js/src/codec/container-id.js new file mode 100644 index 000000000..b22800cfb --- /dev/null +++ b/loro-js/src/codec/container-id.js @@ -0,0 +1,225 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroDecodeError, LoroEncodeError } from "./errors"; +import { readUlebNumber, writeUleb128 } from "./leb128"; +import { PostcardReader, PostcardWriter } from "./postcard"; +import { ContainerType, } from "./types"; +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const textEncoder = new TextEncoder(); +export function unknownContainerType(value) { + assertU8(value, "unknown container type"); + return { kind: "unknown", value }; +} +export function containerTypeToRawByte(type) { + switch (type) { + case ContainerType.Map: + return 0; + case ContainerType.List: + return 1; + case ContainerType.Text: + return 2; + case ContainerType.Tree: + return 3; + case ContainerType.MovableList: + return 4; + case ContainerType.Counter: + return 5; + default: + assertU8(type.value, "unknown container type"); + return type.value; + } +} +export function containerTypeFromRawByte(value) { + switch (value) { + case 0: + return ContainerType.Map; + case 1: + return ContainerType.List; + case 2: + return ContainerType.Text; + case 3: + return ContainerType.Tree; + case 4: + return ContainerType.MovableList; + case 5: + return ContainerType.Counter; + default: + return { kind: "unknown", value }; + } +} +export function containerTypeToHistoricalByte(type) { + switch (type) { + case ContainerType.Text: + return 0; + case ContainerType.Map: + return 1; + case ContainerType.List: + return 2; + case ContainerType.MovableList: + return 3; + case ContainerType.Tree: + return 4; + case ContainerType.Counter: + return 5; + default: + assertU8(type.value, "unknown container type"); + return type.value; + } +} +export function containerTypeFromHistoricalByte(value) { + switch (value) { + case 0: + return ContainerType.Text; + case 1: + return ContainerType.Map; + case 2: + return ContainerType.List; + case 3: + return ContainerType.MovableList; + case 4: + return ContainerType.Tree; + case 5: + return ContainerType.Counter; + default: + return { kind: "unknown", value }; + } +} +export function decodeContainerId(bytes) { + if (bytes.length === 0) { + throw new LoroDecodeError("container ID is empty"); + } + const reader = new ByteReader(bytes); + const first = reader.readU8(); + const containerType = containerTypeFromRawByte(first & 0x7f); + if ((first & 0x80) !== 0) { + const nameOffset = reader.position; + const length = readUlebNumber(reader, 2147483647); + let name; + try { + name = textDecoder.decode(reader.readBytes(length)); + } + catch (error) { + if (error instanceof LoroDecodeError) { + throw error; + } + throw new LoroDecodeError("invalid UTF-8 container name", nameOffset); + } + reader.assertEnd("trailing root container ID bytes"); + return { kind: "root", name, containerType }; + } + if (reader.remaining !== 12) { + throw new LoroDecodeError("invalid normal container ID length", reader.position); + } + const peer = reader.readU64LE(); + const counter = reader.readU32LE() | 0; + reader.assertEnd("trailing normal container ID bytes"); + return { kind: "normal", peer, counter, containerType }; +} +export function encodeContainerId(id) { + const rawType = containerTypeToRawByte(id.containerType); + if (rawType > 0x7f) { + throw new LoroEncodeError(`raw container type is out of range: ${rawType}`); + } + const writer = new ByteWriter(); + if (id.kind === "root") { + const name = textEncoder.encode(id.name); + writer.writeU8(rawType | 0x80); + writeUleb128(writer, name.length); + writer.writeBytes(name); + return writer.toUint8Array(); + } + assertNormalContainerId(id); + writer.writeU8(rawType); + writer.writeU64LE(id.peer); + writer.writeU32LE(id.counter >>> 0); + return writer.toUint8Array(); +} +export function readPostcardContainerId(reader) { + const tag = reader.readU32(); + if (tag === 0) { + return { + kind: "root", + name: reader.readString(), + containerType: containerTypeFromHistoricalByte(reader.readU8()), + }; + } + if (tag === 1) { + return { + kind: "normal", + peer: reader.readU64(), + counter: reader.readI32(), + containerType: containerTypeFromHistoricalByte(reader.readU8()), + }; + } + throw new LoroDecodeError("invalid postcard ContainerID tag", reader.input.position); +} +export function writePostcardContainerId(writer, id) { + if (id.kind === "root") { + writer.writeU32(0); + writer.writeString(id.name); + } + else { + assertNormalContainerId(id); + writer.writeU32(1); + writer.writeU64(id.peer); + writer.writeI32(id.counter); + } + writer.writeU8(containerTypeToHistoricalByte(id.containerType)); +} +export function decodePostcardContainerId(bytes) { + const reader = new PostcardReader(bytes); + const id = readPostcardContainerId(reader); + reader.assertEnd(); + return id; +} +export function encodePostcardContainerId(id) { + const writer = new PostcardWriter(); + writePostcardContainerId(writer, id); + return writer.toUint8Array(); +} +export function readPostcardOptionalContainerId(reader) { + const tag = reader.readU32(); + if (tag === 0) { + return undefined; + } + if (tag === 1) { + return readPostcardContainerId(reader); + } + throw new LoroDecodeError("invalid postcard Option tag", reader.input.position); +} +export function writePostcardOptionalContainerId(writer, id) { + if (id === undefined) { + writer.writeU32(0); + return; + } + writer.writeU32(1); + writePostcardContainerId(writer, id); +} +export function decodePostcardOptionalContainerId(bytes) { + const reader = new PostcardReader(bytes); + const id = readPostcardOptionalContainerId(reader); + reader.assertEnd(); + return id; +} +export function encodePostcardOptionalContainerId(id) { + const writer = new PostcardWriter(); + writePostcardOptionalContainerId(writer, id); + return writer.toUint8Array(); +} +function assertNormalContainerId(id) { + if (id.peer < 0n || id.peer > 0xffffffffffffffffn) { + throw new LoroEncodeError(`container peer ID is out of range: ${id.peer}`); + } + if (!Number.isSafeInteger(id.counter) || + id.counter < -2147483648 || + id.counter > 2147483647) { + throw new LoroEncodeError(`container counter is out of range: ${id.counter}`); + } +} +function assertU8(value, name) { + if (!Number.isSafeInteger(value) || value < 0 || value > 0xff) { + throw new LoroEncodeError(`${name} is out of range: ${value}`); + } +} +export function isKnownContainerType(type) { + return typeof type === "string"; +} diff --git a/loro-js/src/codec/document.js b/loro-js/src/codec/document.js new file mode 100644 index 000000000..41d3d7a53 --- /dev/null +++ b/loro-js/src/codec/document.js @@ -0,0 +1,115 @@ +import { ByteReader, ByteWriter, bytesEqual } from "./bytes"; +import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; +import { readUlebNumber, writeUleb128 } from "./leb128"; +import { LORO_XXHASH_SEED, xxhash32 } from "./xxhash32"; +const DOCUMENT_MAGIC = Uint8Array.of(0x6c, 0x6f, 0x72, 0x6f); +const DOCUMENT_HEADER_LENGTH = 22; +export var EncodeMode; +(function (EncodeMode) { + EncodeMode[EncodeMode["FastSnapshot"] = 3] = "FastSnapshot"; + EncodeMode[EncodeMode["FastUpdates"] = 4] = "FastUpdates"; +})(EncodeMode || (EncodeMode = {})); +export function decodeDocument(bytes, options = {}) { + const reader = new ByteReader(bytes); + decodeAssert(bytes.length >= DOCUMENT_HEADER_LENGTH, "document is too short", 0); + decodeAssert(bytesEqual(reader.readBytes(4), DOCUMENT_MAGIC), "invalid document magic", 0); + const checksumPrefix = reader.readBytes(12); + if (options.requireCanonicalPrefix === true) { + for (const byte of checksumPrefix) { + decodeAssert(byte === 0, "nonzero document checksum prefix", 4); + } + } + const checksum = reader.readU32LE(); + const rawMode = reader.readU16BE(); + if (rawMode !== EncodeMode.FastSnapshot && rawMode !== EncodeMode.FastUpdates) { + throw new LoroDecodeError(`unsupported document mode ${rawMode}`, 20); + } + const body = reader.readRemaining(); + if (options.checkChecksum !== false) { + const expected = xxhash32(bytes.subarray(20), LORO_XXHASH_SEED); + decodeAssert(checksum === expected, "document checksum mismatch", 16); + } + return { mode: rawMode, body, checksum, checksumPrefix }; +} +export function encodeDocument(mode, body) { + if (mode !== EncodeMode.FastSnapshot && mode !== EncodeMode.FastUpdates) { + throw new LoroEncodeError(`unsupported document mode ${mode}`); + } + // Header and body share one allocation so the body is copied exactly once; + // the checksum covers the mode and body bytes and is backfilled afterwards. + const output = new Uint8Array(DOCUMENT_HEADER_LENGTH + body.length); + output.set(DOCUMENT_MAGIC, 0); + // Bytes 4..16 stay zero as the checksum prefix. + output[20] = mode >>> 8; + output[21] = mode & 0xff; + output.set(body, DOCUMENT_HEADER_LENGTH); + const checksum = xxhash32(output.subarray(20), LORO_XXHASH_SEED); + output[16] = checksum & 0xff; + output[17] = (checksum >>> 8) & 0xff; + output[18] = (checksum >>> 16) & 0xff; + output[19] = (checksum >>> 24) & 0xff; + return output; +} +export function decodeFastSnapshotBody(body) { + const reader = new ByteReader(body); + const oplog = readU32LengthPrefixed(reader, "oplog"); + const state = readU32LengthPrefixed(reader, "state"); + const shallowRootState = readU32LengthPrefixed(reader, "shallow root state"); + reader.assertEnd("trailing FastSnapshot bytes"); + return { oplog, state, shallowRootState }; +} +export function encodeFastSnapshotBody(snapshot) { + const writer = new ByteWriter(12 + snapshot.oplog.length + snapshot.state.length + snapshot.shallowRootState.length); + writeU32LengthPrefixed(writer, snapshot.oplog, "oplog"); + writeU32LengthPrefixed(writer, snapshot.state, "state"); + writeU32LengthPrefixed(writer, snapshot.shallowRootState, "shallow root state"); + return writer.toUint8Array(); +} +export function decodeFastUpdatesBody(body) { + const reader = new ByteReader(body); + const blocks = []; + while (reader.remaining > 0) { + const length = readUlebNumber(reader, 4294967295); + blocks.push(reader.readBytes(length)); + } + return blocks; +} +export function encodeFastUpdatesBody(blocks) { + const writer = new ByteWriter(); + for (const block of blocks) { + writeUleb128(writer, block.length); + writer.writeBytes(block); + } + return writer.toUint8Array(); +} +export function decodeFastSnapshot(bytes, options) { + const document = decodeDocument(bytes, options); + decodeAssert(document.mode === EncodeMode.FastSnapshot, "document is not a FastSnapshot", 20); + return decodeFastSnapshotBody(document.body); +} +export function encodeFastSnapshot(snapshot) { + return encodeDocument(EncodeMode.FastSnapshot, encodeFastSnapshotBody(snapshot)); +} +export function decodeFastUpdates(bytes, options) { + const document = decodeDocument(bytes, options); + decodeAssert(document.mode === EncodeMode.FastUpdates, "document is not FastUpdates", 20); + return decodeFastUpdatesBody(document.body); +} +export function encodeFastUpdates(blocks) { + return encodeDocument(EncodeMode.FastUpdates, encodeFastUpdatesBody(blocks)); +} +function readU32LengthPrefixed(reader, label) { + const offset = reader.position; + const length = reader.readU32LE(); + if (length > reader.remaining) { + throw new LoroDecodeError(`${label} length exceeds remaining input`, offset); + } + return reader.readBytes(length); +} +function writeU32LengthPrefixed(writer, value, label) { + if (value.length > 4294967295) { + throw new LoroEncodeError(`${label} is too large`); + } + writer.writeU32LE(value.length); + writer.writeBytes(value); +} diff --git a/loro-js/src/codec/errors.js b/loro-js/src/codec/errors.js new file mode 100644 index 000000000..447b2d98f --- /dev/null +++ b/loro-js/src/codec/errors.js @@ -0,0 +1,24 @@ +export class LoroDecodeError extends Error { + offset; + constructor(message, offset) { + super(offset === undefined ? message : `${message} at byte ${offset}`); + this.name = "LoroDecodeError"; + this.offset = offset; + } +} +export class LoroEncodeError extends Error { + constructor(message) { + super(message); + this.name = "LoroEncodeError"; + } +} +export function decodeAssert(condition, message, offset) { + if (!condition) { + throw new LoroDecodeError(message, offset); + } +} +export function encodeAssert(condition, message) { + if (!condition) { + throw new LoroEncodeError(message); + } +} diff --git a/loro-js/src/codec/id.js b/loro-js/src/codec/id.js new file mode 100644 index 000000000..7682d7abc --- /dev/null +++ b/loro-js/src/codec/id.js @@ -0,0 +1,53 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroDecodeError, LoroEncodeError } from "./errors"; +import { PostcardReader, PostcardWriter } from "./postcard"; +const I32_MIN = -2147483648; +const I32_MAX = 2147483647; +const U64_MAX = 0xffffffffffffffffn; +export function decodeChangeBlockKey(bytes) { + if (bytes.length !== 12) { + throw new LoroDecodeError("invalid change block key length"); + } + const reader = new ByteReader(bytes); + const peer = reader.readU64BE(); + const counter = reader.readU32BE() | 0; + reader.assertEnd("trailing change block key bytes"); + return { peer, counter }; +} +export function encodeChangeBlockKey(id) { + assertId(id); + const writer = new ByteWriter(12); + writer.writeU64BE(id.peer); + writer.writeU32BE(id.counter >>> 0); + return writer.toUint8Array(); +} +export function readPostcardId(reader) { + return { + peer: reader.readU64(), + counter: reader.readI32(), + }; +} +export function writePostcardId(writer, id) { + assertId(id); + writer.writeU64(id.peer); + writer.writeI32(id.counter); +} +export function decodePostcardId(bytes) { + const reader = new PostcardReader(bytes); + const id = readPostcardId(reader); + reader.assertEnd(); + return id; +} +export function encodePostcardId(id) { + const writer = new PostcardWriter(); + writePostcardId(writer, id); + return writer.toUint8Array(); +} +export function assertId(id) { + if (id.peer < 0n || id.peer > U64_MAX) { + throw new LoroEncodeError(`peer ID is out of range: ${id.peer}`); + } + if (!Number.isSafeInteger(id.counter) || id.counter < I32_MIN || id.counter > I32_MAX) { + throw new LoroEncodeError(`ID counter is out of range: ${id.counter}`); + } +} diff --git a/loro-js/src/codec/index.js b/loro-js/src/codec/index.js new file mode 100644 index 000000000..a9a320222 --- /dev/null +++ b/loro-js/src/codec/index.js @@ -0,0 +1,20 @@ +export * from "./bytes"; +export * from "./change-block"; +export * from "./change-block-codec"; +export * from "./change-block-tables"; +export * from "./change-value"; +export * from "./container-id"; +export * from "./document"; +export * from "./errors"; +export * from "./id"; +export * from "./leb128"; +export * from "./lz4"; +export * from "./postcard"; +export * from "./position-arena"; +export { decodeAnyRleI32, decodeAnyRleU32, decodeAnyRleU64, decodeAnyRleUsize, decodeBoolRle, decodeColumnarVec, decodeColumnarVecMaybeWrapped, decodeDeltaOfDeltaI64, decodeDeltaRleI32, decodeDeltaRleIsize, decodeDeltaRleU32, decodeDeltaRleUsize, decodeRleU8, decodeRleU32, encodeAnyRleI32, encodeAnyRleU32, encodeAnyRleU64, encodeAnyRleUsize, encodeBoolRle, encodeColumnarVec, encodeColumnarVecWrapped, encodeDeltaOfDeltaI64, encodeDeltaRleI32, encodeDeltaRleIsize, encodeDeltaRleU32, encodeDeltaRleUsize, encodeRleU8, encodeRleU32, takeAnyRleI32, takeAnyRleU32, takeAnyRleUsize, takeBoolRle, takeColumnarVec, takeDeltaOfDeltaI64, } from "./serde-columnar"; +export * from "./sstable"; +export * from "./state-snapshot"; +export * from "./types"; +export * from "./value"; +export * from "./version"; +export * from "./xxhash32"; diff --git a/loro-js/src/codec/leb128.js b/loro-js/src/codec/leb128.js new file mode 100644 index 000000000..1076723c4 --- /dev/null +++ b/loro-js/src/codec/leb128.js @@ -0,0 +1,149 @@ +import { LoroDecodeError, LoroEncodeError } from "./errors"; +export const U64_MAX = 0xffffffffffffffffn; +export const I64_MIN = -0x8000000000000000n; +export const I64_MAX = 0x7fffffffffffffffn; +export function readUleb128(reader, max = U64_MAX) { + let result = 0n; + let shift = 0n; + const start = reader.position; + for (let index = 0; index < 10; index += 1) { + const byte = reader.readU8(); + const payload = BigInt(byte & 0x7f); + if (shift === 63n && payload > 1n) { + throw new LoroDecodeError("ULEB128 overflow", start); + } + result |= payload << shift; + if ((byte & 0x80) === 0) { + if (result > max) { + throw new LoroDecodeError("ULEB128 value is out of range", start); + } + return result; + } + shift += 7n; + } + throw new LoroDecodeError("ULEB128 overflow", start); +} +export function readUlebNumber(reader, max = Number.MAX_SAFE_INTEGER) { + if (!Number.isSafeInteger(max) || max < 0) { + throw new LoroDecodeError("invalid ULEB128 number limit", reader.position); + } + const start = reader.position; + let result = 0; + let multiplier = 1; + // Set once the value provably exceeds Number.MAX_SAFE_INTEGER (and therefore + // any max). Bytes are still consumed with readUleb128's overflow rules so + // the reader position and error behavior match the BigInt implementation. + let exceedsSafeInteger = false; + for (let index = 0; index < 10; index += 1) { + const byte = reader.readU8(); + const payload = byte & 0x7f; + if (index === 9 && payload > 1) { + throw new LoroDecodeError("ULEB128 overflow", start); + } + if (!exceedsSafeInteger && payload !== 0) { + if (multiplier > Number.MAX_SAFE_INTEGER / payload) { + exceedsSafeInteger = true; + } + else { + const product = payload * multiplier; + if (result > Number.MAX_SAFE_INTEGER - product) { + exceedsSafeInteger = true; + } + else { + result += product; + } + } + } + if ((byte & 0x80) === 0) { + if (exceedsSafeInteger || result > max) { + throw new LoroDecodeError("ULEB128 value is out of range", start); + } + return result; + } + multiplier *= 128; + } + throw new LoroDecodeError("ULEB128 overflow", start); +} +export function writeUleb128(writer, input) { + if (typeof input === "number" && + Number.isSafeInteger(input) && + input >= 0 && + input < 2147483648) { + // Smi fast path: encode small numbers without any BigInt arithmetic. + let value = input; + do { + let byte = value & 0x7f; + value >>>= 7; + if (value !== 0) { + byte |= 0x80; + } + writer.writeU8(byte); + } while (value !== 0); + return; + } + let value = typeof input === "number" ? numberToBigInt(input, "ULEB128") : input; + if (value < 0n || value > U64_MAX) { + throw new LoroEncodeError(`ULEB128 value is out of range: ${value}`); + } + do { + let byte = Number(value & 0x7fn); + value >>= 7n; + if (value !== 0n) { + byte |= 0x80; + } + writer.writeU8(byte); + } while (value !== 0n); +} +export function readSleb128(reader) { + let result = 0n; + let shift = 0n; + let byte = 0; + const start = reader.position; + for (let index = 0; index < 10; index += 1) { + byte = reader.readU8(); + const payload = BigInt(byte & 0x7f); + result |= payload << shift; + shift += 7n; + if ((byte & 0x80) === 0) { + if (index === 9) { + if (byte !== 0 && byte !== 0x7f) { + throw new LoroDecodeError("SLEB128 overflow", start); + } + return BigInt.asIntN(64, result); + } + if ((byte & 0x40) !== 0) { + result |= -1n << shift; + } + if (result < I64_MIN || result > I64_MAX) { + throw new LoroDecodeError("SLEB128 value is out of range", start); + } + return result; + } + } + throw new LoroDecodeError("SLEB128 overflow", start); +} +export function writeSleb128(writer, input) { + let value = typeof input === "number" ? numberToBigInt(input, "SLEB128") : input; + if (value < I64_MIN || value > I64_MAX) { + throw new LoroEncodeError(`SLEB128 value is out of range: ${value}`); + } + for (;;) { + let byte = Number(value & 0x7fn); + value >>= 7n; + const sign = (byte & 0x40) !== 0; + const done = (value === 0n && !sign) || (value === -1n && sign); + if (!done) { + byte |= 0x80; + } + writer.writeU8(byte); + if (done) { + return; + } + } +} +function numberToBigInt(value, name) { + if (!Number.isSafeInteger(value)) { + throw new LoroEncodeError(`${name} number must be a safe integer: ${value}`); + } + return BigInt(value); +} diff --git a/loro-js/src/codec/lz4.js b/loro-js/src/codec/lz4.js new file mode 100644 index 000000000..bdf4f6761 --- /dev/null +++ b/loro-js/src/codec/lz4.js @@ -0,0 +1,307 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; +import { xxhash32 } from "./xxhash32"; +const LZ4_MAGIC = 407708164; +const WINDOW_SIZE = 65536; +const MAX_MATCH_DISTANCE = 0xffff; +const MIN_MATCH_LENGTH = 4; +const LAST_LITERALS = 5; +const MATCH_FIND_LIMIT = 12; +const HASH_LOG = 16; +const MIN_HASH_LOG = 10; +const HASH_MULTIPLIER = 2654435761; +// Encoding runs synchronously, so every block reuses one table sized for the +// largest hash log instead of allocating a fresh table per block. +const sharedHashTable = new Uint32Array(1 << HASH_LOG); +export function decodeLz4Frame(bytes, options = {}) { + const reader = new ByteReader(bytes); + decodeAssert(reader.readU32LE() === LZ4_MAGIC, "invalid LZ4 frame magic", 0); + const descriptorStart = reader.position; + const flags = reader.readU8(); + const version = (flags >>> 6) & 0x03; + decodeAssert(version === 1, "unsupported LZ4 frame version", descriptorStart); + decodeAssert((flags & 0x02) === 0, "reserved LZ4 FLG bit is set", descriptorStart); + const independentBlocks = (flags & 0x20) !== 0; + const blockChecksum = (flags & 0x10) !== 0; + const hasContentSize = (flags & 0x08) !== 0; + const contentChecksum = (flags & 0x04) !== 0; + const hasDictionary = (flags & 0x01) !== 0; + const blockDescriptor = reader.readU8(); + const maxBlockSize = decodeBlockMaximum(blockDescriptor, reader.position - 1); + let expectedContentSize; + if (hasContentSize) { + expectedContentSize = reader.readU64LE(); + } + if (hasDictionary) { + reader.readU32LE(); + throw new LoroDecodeError("LZ4 dictionaries are unsupported", descriptorStart); + } + const headerChecksumPosition = reader.position; + const storedHeaderChecksum = reader.readU8(); + if (options.checkChecksum !== false) { + const descriptor = bytes.subarray(descriptorStart, headerChecksumPosition); + const expected = (xxhash32(descriptor, 0) >>> 8) & 0xff; + decodeAssert(storedHeaderChecksum === expected, "LZ4 header checksum mismatch", headerChecksumPosition); + } + if (options.requireCanonicalProfile === true) { + decodeAssert(flags === 0x60, "noncanonical LZ4 frame flags", descriptorStart); + } + const output = new Lz4Output(); + for (;;) { + const infoOffset = reader.position; + const blockInfo = reader.readU32LE(); + if (blockInfo === 0) { + break; + } + const raw = (blockInfo & 2147483648) !== 0; + const length = blockInfo & 2147483647; + decodeAssert(length <= maxBlockSize, "LZ4 data block exceeds BD maximum", infoOffset); + const block = reader.readBytes(length); + if (blockChecksum) { + const stored = reader.readU32LE(); + if (options.checkChecksum !== false) { + decodeAssert(stored === xxhash32(block, 0), "LZ4 block checksum mismatch", reader.position - 4); + } + } + const blockOutputStart = output.length; + if (raw) { + output.ensureCapacity(block.length); + output.writeBytes(block); + } + else { + output.ensureCapacity(maxBlockSize); + const prefixStart = independentBlocks + ? blockOutputStart + : Math.max(0, blockOutputStart - WINDOW_SIZE); + decodeLz4BlockInto(block, output, prefixStart, blockOutputStart + maxBlockSize); + } + decodeAssert(output.length - blockOutputStart <= maxBlockSize, "LZ4 data block output exceeds BD maximum", infoOffset); + } + const result = output.finish(); + if (expectedContentSize !== undefined) { + decodeAssert(expectedContentSize === BigInt(result.length), "LZ4 content size mismatch", descriptorStart); + } + if (contentChecksum) { + const stored = reader.readU32LE(); + if (options.checkChecksum !== false) { + decodeAssert(stored === xxhash32(result, 0), "LZ4 content checksum mismatch", reader.position - 4); + } + } + reader.assertEnd("trailing LZ4 frame bytes"); + return result; +} +export function encodeLz4FrameRaw(bytes) { + const { descriptor, maxBlockSize } = chooseBlockDescriptor(bytes.length); + const flags = 0x60; + const header = Uint8Array.of(flags, descriptor); + const writer = new ByteWriter(bytes.length + 16 + Math.ceil(bytes.length / maxBlockSize) * 4); + writer.writeU32LE(LZ4_MAGIC); + writer.writeBytes(header); + writer.writeU8((xxhash32(header, 0) >>> 8) & 0xff); + for (let offset = 0; offset < bytes.length; offset += maxBlockSize) { + const block = bytes.subarray(offset, Math.min(bytes.length, offset + maxBlockSize)); + const compressed = encodeLz4Block(block); + if (compressed.length < block.length) { + writer.writeU32LE(compressed.length); + writer.writeBytes(compressed); + } + else { + writer.writeU32LE((2147483648 + block.length) >>> 0); + writer.writeBytes(block); + } + } + writer.writeU32LE(0); + return writer.toUint8Array(); +} +function encodeLz4Block(input) { + if (input.length < MATCH_FIND_LIMIT + 1) { + return encodeLastLiterals(input); + } + const writer = new ByteWriter(input.length); + const hashLog = Math.min(HASH_LOG, Math.max(MIN_HASH_LOG, Math.ceil(Math.log2(input.length)))); + const hashShift = 32 - hashLog; + const hashTable = sharedHashTable; + hashTable.fill(0, 0, 1 << hashLog); + const lastMatchStart = input.length - MATCH_FIND_LIMIT; + const matchLimit = input.length - LAST_LITERALS; + let anchor = 0; + let position = 0; + while (position <= lastMatchStart) { + const hash = hashSequence(input, position, hashShift); + const candidateEntry = hashTable[hash]; + hashTable[hash] = position + 1; + const candidate = candidateEntry - 1; + if (candidateEntry === 0 || + position - candidate > MAX_MATCH_DISTANCE || + !equalFourBytes(input, candidate, position)) { + position += 1; + continue; + } + let matchEnd = position + MIN_MATCH_LENGTH; + let candidateEnd = candidate + MIN_MATCH_LENGTH; + while (matchEnd < matchLimit && input[candidateEnd] === input[matchEnd]) { + matchEnd += 1; + candidateEnd += 1; + } + writeSequence(writer, input.subarray(anchor, position), position - candidate, matchEnd - position); + for (let index = position + 1; index < matchEnd; index += 1) { + if (index + MIN_MATCH_LENGTH > input.length) { + break; + } + hashTable[hashSequence(input, index, hashShift)] = index + 1; + } + position = matchEnd; + anchor = matchEnd; + } + writeLastLiterals(writer, input.subarray(anchor)); + return writer.toUint8Array(); +} +function encodeLastLiterals(input) { + const writer = new ByteWriter(input.length + 2); + writeLastLiterals(writer, input); + return writer.toUint8Array(); +} +function writeSequence(writer, literals, offset, matchLength) { + const encodedMatchLength = matchLength - MIN_MATCH_LENGTH; + writer.writeU8((Math.min(literals.length, 15) << 4) | Math.min(encodedMatchLength, 15)); + writeExtendedLength(writer, literals.length); + writer.writeBytes(literals); + writer.writeU16LE(offset); + writeExtendedLength(writer, encodedMatchLength); +} +function writeLastLiterals(writer, literals) { + writer.writeU8(Math.min(literals.length, 15) << 4); + writeExtendedLength(writer, literals.length); + writer.writeBytes(literals); +} +function writeExtendedLength(writer, length) { + if (length < 15) { + return; + } + let remaining = length - 15; + while (remaining >= 255) { + writer.writeU8(255); + remaining -= 255; + } + writer.writeU8(remaining); +} +function hashSequence(input, offset, hashShift) { + const sequence = (input[offset] | + (input[offset + 1] << 8) | + (input[offset + 2] << 16) | + (input[offset + 3] << 24)) >>> + 0; + return Math.imul(sequence, HASH_MULTIPLIER) >>> hashShift; +} +function equalFourBytes(input, left, right) { + return (input[left] === input[right] && + input[left + 1] === input[right + 1] && + input[left + 2] === input[right + 2] && + input[left + 3] === input[right + 3]); +} +function decodeLz4BlockInto(input, output, prefixStart, outputLimit) { + const reader = new ByteReader(input); + while (reader.remaining > 0) { + const token = reader.readU8(); + const literalLength = readExtendedLength(reader, token >>> 4); + decodeAssert(literalLength <= outputLimit - output.length, "LZ4 data block output exceeds BD maximum", reader.position); + output.writeBytes(reader.readBytes(literalLength)); + if (reader.remaining === 0) { + return; + } + const offsetPosition = reader.position; + const offset = reader.readU16LE(); + decodeAssert(offset !== 0, "zero LZ4 match offset", offsetPosition); + decodeAssert(offset <= output.length - prefixStart, "LZ4 match offset is out of bounds", offsetPosition); + const matchLength = 4 + readExtendedLength(reader, token & 0x0f); + decodeAssert(matchLength <= outputLimit - output.length, "LZ4 data block output exceeds BD maximum", reader.position); + // Capacity for the whole block was reserved before decoding started and + // the offset checks above keep the source range inside the output, so the + // copy can read and write the buffer directly. + const bytes = output.bytes; + let length = output.length; + if (offset >= matchLength) { + bytes.copyWithin(length, length - offset, length - offset + matchLength); + length += matchLength; + } + else { + // Overlapping matches reference bytes written by this same copy. + for (let index = 0; index < matchLength; index += 1) { + bytes[length] = bytes[length - offset]; + length += 1; + } + } + output.length = length; + } +} +class Lz4Output { + bytes = new Uint8Array(); + length = 0; + ensureCapacity(extra) { + const required = this.length + extra; + if (required <= this.bytes.length) { + return; + } + let capacity = Math.max(64, this.bytes.length); + while (capacity < required) { + capacity = Math.max(required, capacity * 2); + } + const next = new Uint8Array(capacity); + next.set(this.bytes.subarray(0, this.length)); + this.bytes = next; + } + writeBytes(bytes) { + this.ensureCapacity(bytes.length); + this.bytes.set(bytes, this.length); + this.length += bytes.length; + } + finish() { + return this.length === this.bytes.length + ? this.bytes + : this.bytes.slice(0, this.length); + } +} +function readExtendedLength(reader, base) { + let length = base; + if (base !== 15) { + return length; + } + for (;;) { + const extra = reader.readU8(); + length += extra; + decodeAssert(Number.isSafeInteger(length), "LZ4 sequence length overflow", reader.position - 1); + if (extra !== 255) { + return length; + } + } +} +function decodeBlockMaximum(descriptor, offset) { + decodeAssert((descriptor & 0x8f) === 0, "invalid LZ4 BD byte", offset); + switch ((descriptor >>> 4) & 0x07) { + case 4: + return 64 * 1024; + case 5: + return 256 * 1024; + case 6: + return 1024 * 1024; + case 7: + return 4 * 1024 * 1024; + default: + throw new LoroDecodeError("unsupported LZ4 block maximum", offset); + } +} +function chooseBlockDescriptor(length) { + if (!Number.isSafeInteger(length) || length < 0) { + throw new LoroEncodeError("invalid LZ4 input length"); + } + if (length <= 64 * 1024) { + return { descriptor: 0x40, maxBlockSize: 64 * 1024 }; + } + if (length <= 256 * 1024) { + return { descriptor: 0x50, maxBlockSize: 256 * 1024 }; + } + if (length <= 1024 * 1024) { + return { descriptor: 0x60, maxBlockSize: 1024 * 1024 }; + } + return { descriptor: 0x70, maxBlockSize: 4 * 1024 * 1024 }; +} diff --git a/loro-js/src/codec/position-arena.js b/loro-js/src/codec/position-arena.js new file mode 100644 index 000000000..d1a5660a0 --- /dev/null +++ b/loro-js/src/codec/position-arena.js @@ -0,0 +1,67 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroEncodeError, decodeAssert } from "./errors"; +import { readUlebNumber, writeUleb128 } from "./leb128"; +import { decodeAnyRleUsize, decodeColumnarVecMaybeWrapped, encodeAnyRleUsize, encodeColumnarVecWrapped, } from "./serde-columnar"; +export function decodePositionArena(bytes) { + if (bytes.length === 0) { + return []; + } + const columns = decodeColumnarVecMaybeWrapped(bytes); + decodeAssert(columns.length === 2, "position arena must have two columns"); + const commonPrefixes = decodeAnyRleUsize(columns[0]); + const reader = new ByteReader(columns[1]); + const count = readUlebNumber(reader, 10000000); + decodeAssert(count === commonPrefixes.length, "position arena column length mismatch"); + const positions = []; + let previous = new Uint8Array(); + let previousLength = 0; + for (const commonBigInt of commonPrefixes) { + // Number comparison is exact here: accepted values are at most + // previousLength (an array length), so Number(commonBigInt) is precise + // whenever the assertion passes, and larger BigInts convert to values + // greater than previousLength and fail the same way. + const common = Number(commonBigInt); + decodeAssert(common <= previousLength, "invalid position arena common prefix"); + const suffix = reader.readBytes(readUlebNumber(reader, 2147483647)); + const position = new Uint8Array(common + suffix.length); + position.set(previous.subarray(0, common)); + position.set(suffix, common); + positions.push(position); + previous = position; + previousLength = position.length; + } + reader.assertEnd("trailing position arena bytes"); + return positions; +} +export function encodePositionArena(positions, options = {}) { + if (positions.length === 0 && options.encodeEmpty !== true) { + return new Uint8Array(); + } + if (positions.length > 10000000) { + throw new LoroEncodeError(`position arena is too large: ${positions.length}`); + } + const commonPrefixes = []; + const suffixes = new ByteWriter(); + writeUleb128(suffixes, positions.length); + let previous = new Uint8Array(); + for (const position of positions) { + const common = commonPrefixLength(previous, position); + commonPrefixes.push(BigInt(common)); + const suffix = position.subarray(common); + writeUleb128(suffixes, suffix.length); + suffixes.writeBytes(suffix); + previous = position; + } + return encodeColumnarVecWrapped([ + encodeAnyRleUsize(commonPrefixes), + suffixes.toUint8Array(), + ]); +} +function commonPrefixLength(left, right) { + const length = Math.min(left.length, right.length); + let index = 0; + while (index < length && left[index] === right[index]) { + index += 1; + } + return index; +} diff --git a/loro-js/src/codec/postcard.js b/loro-js/src/codec/postcard.js new file mode 100644 index 000000000..d2a61affa --- /dev/null +++ b/loro-js/src/codec/postcard.js @@ -0,0 +1,172 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroDecodeError, LoroEncodeError } from "./errors"; +import { I64_MAX, I64_MIN, U64_MAX, readUleb128, readUlebNumber, writeUleb128, } from "./leb128"; +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const textEncoder = new TextEncoder(); +// Shared scratch for f64 I/O, avoiding a Uint8Array + DataView allocation +// per call. Never returned to callers. +const f64Scratch = new Uint8Array(8); +const f64View = new DataView(f64Scratch.buffer); +export class PostcardReader { + input; + constructor(input) { + this.input = input instanceof ByteReader ? input : new ByteReader(input); + } + readU8() { + return this.input.readU8(); + } + readU16() { + return readUlebNumber(this.input, 0xffff); + } + readU32() { + return readUlebNumber(this.input, 4294967295); + } + readU64() { + return readUleb128(this.input, U64_MAX); + } + readUsize() { + return readUlebNumber(this.input); + } + readI32() { + const value = this.readSigned(32); + return Number(value); + } + readI64() { + return this.readSigned(64); + } + readBool() { + const value = this.input.readU8(); + if (value === 0) { + return false; + } + if (value === 1) { + return true; + } + throw new LoroDecodeError("invalid postcard boolean", this.input.position - 1); + } + readF64() { + f64Scratch.set(this.input.readBytes(8)); + return f64View.getFloat64(0, true); + } + readBytes() { + return this.input.readBytes(this.readUsize()); + } + readString() { + const offset = this.input.position; + try { + return textDecoder.decode(this.readBytes()); + } + catch { + throw new LoroDecodeError("invalid postcard UTF-8 string", offset); + } + } + readOption(readValue) { + const tag = this.input.readU8(); + if (tag === 0) { + return undefined; + } + if (tag === 1) { + return readValue(this); + } + throw new LoroDecodeError("invalid postcard option tag", this.input.position - 1); + } + readArray(readValue) { + const length = this.readUsize(); + // push() keeps the array PACKED; new Array(length) + index assignment + // would produce a permanently holey elements kind. + const values = []; + for (let index = 0; index < length; index += 1) { + values.push(readValue(this, index)); + } + return values; + } + assertEnd() { + this.input.assertEnd("trailing postcard bytes"); + } + readSigned(bits) { + const encoded = readUleb128(this.input, bits === 32 ? 0xffffffffn : U64_MAX); + const value = (encoded >> 1n) ^ -(encoded & 1n); + const min = bits === 32 ? -0x80000000n : I64_MIN; + const max = bits === 32 ? 0x7fffffffn : I64_MAX; + if (value < min || value > max) { + throw new LoroDecodeError("postcard signed integer is out of range"); + } + return value; + } +} +export class PostcardWriter { + output; + constructor(output = new ByteWriter()) { + this.output = output; + } + writeU8(value) { + this.output.writeU8(value); + } + writeU16(value) { + this.writeUnsignedNumber(value, 0xffff, "u16"); + } + writeU32(value) { + this.writeUnsignedNumber(value, 4294967295, "u32"); + } + writeU64(value) { + writeUleb128(this.output, value); + } + writeUsize(value) { + this.writeUnsignedNumber(value, Number.MAX_SAFE_INTEGER, "usize"); + } + writeI32(value) { + if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { + throw new LoroEncodeError(`i32 is out of range: ${value}`); + } + this.writeSigned(BigInt(value), 32); + } + writeI64(value) { + this.writeSigned(value, 64); + } + writeBool(value) { + this.output.writeU8(value ? 1 : 0); + } + writeF64(value) { + f64View.setFloat64(0, value, true); + this.output.writeBytes(f64Scratch); + } + writeBytes(value) { + this.writeUsize(value.length); + this.output.writeBytes(value); + } + writeString(value) { + this.writeBytes(textEncoder.encode(value)); + } + writeOption(value, writeValue) { + if (value === undefined) { + this.output.writeU8(0); + return; + } + this.output.writeU8(1); + writeValue(this, value); + } + writeArray(values, writeValue) { + this.writeUsize(values.length); + for (const value of values) { + writeValue(this, value); + } + } + toUint8Array() { + return this.output.toUint8Array(); + } + writeUnsignedNumber(value, max, label) { + if (!Number.isSafeInteger(value) || value < 0 || value > max) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } + writeUleb128(this.output, value); + } + writeSigned(value, bits) { + const min = bits === 32 ? -0x80000000n : I64_MIN; + const max = bits === 32 ? 0x7fffffffn : I64_MAX; + if (value < min || value > max) { + throw new LoroEncodeError(`i${bits} is out of range: ${value}`); + } + const encoded = (value << 1n) ^ (value >> BigInt(bits - 1)); + writeUleb128(this.output, encoded); + } +} diff --git a/loro-js/src/codec/serde-columnar.js b/loro-js/src/codec/serde-columnar.js new file mode 100644 index 000000000..a2fa0d375 --- /dev/null +++ b/loro-js/src/codec/serde-columnar.js @@ -0,0 +1,902 @@ +import { ByteReader, ByteWriter } from "./bytes"; +import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; +import { I64_MAX, I64_MIN, U64_MAX, readUleb128, readUlebNumber, writeUleb128 } from "./leb128"; +const U32_MAX = 4294967295; +const I32_MIN = -2147483648; +const I32_MAX = 2147483647; +const I128_MIN = -(1n << 127n); +const I128_MAX = (1n << 127n) - 1n; +const U128_MAX = (1n << 128n) - 1n; +const MAX_COLUMN_VALUES = 10000000; +const MIN_SAFE_BIGINT = BigInt(-Number.MAX_SAFE_INTEGER); +const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); +export function decodeColumnarVec(bytes) { + const reader = new ByteReader(bytes); + const columns = readColumnarVec(reader); + reader.assertEnd("trailing serde-columnar bytes"); + return columns; +} +export function takeColumnarVec(bytes) { + const reader = new ByteReader(bytes); + const columns = readColumnarVec(reader); + return [columns, bytes.subarray(reader.position)]; +} +export function decodeColumnarVecMaybeWrapped(bytes) { + if (bytes.length === 0) { + return []; + } + const reader = new ByteReader(bytes); + if (readUlebNumber(reader, MAX_COLUMN_VALUES) === 1) { + const columns = readColumnarVec(reader); + reader.assertEnd("trailing wrapped serde-columnar bytes"); + return columns; + } + return decodeColumnarVec(bytes); +} +export function encodeColumnarVec(columns) { + const writer = new ByteWriter(); + writeUleb128(writer, columns.length); + for (const column of columns) { + writeUleb128(writer, column.length); + writer.writeBytes(column); + } + return writer.toUint8Array(); +} +export function encodeColumnarVecWrapped(columns) { + const writer = new ByteWriter(); + writeUleb128(writer, 1); + writer.writeBytes(encodeColumnarVec(columns)); + return writer.toUint8Array(); +} +export function decodeBoolRle(bytes) { + const reader = new ByteReader(bytes); + const values = []; + let state = false; + while (reader.remaining > 0) { + const length = readUlebNumber(reader, MAX_COLUMN_VALUES); + assertDecodedLength(values.length, length, "bool RLE"); + for (let index = 0; index < length; index += 1) { + values.push(state); + } + state = !state; + } + return values; +} +export function encodeBoolRle(values) { + const writer = new ByteWriter(); + if (values.length === 0) { + return writer.toUint8Array(); + } + let state = false; + let runLength = 0; + for (const value of values) { + if (value === state) { + runLength += 1; + } + else { + writeUleb128(writer, runLength); + state = !state; + runLength = 1; + } + } + writeUleb128(writer, runLength); + return writer.toUint8Array(); +} +export function takeBoolRle(bytes, count) { + assertCount(count); + const reader = new ByteReader(bytes); + const values = []; + let state = false; + while (values.length < count) { + decodeAssert(reader.remaining > 0, "bool RLE has too few elements", reader.position); + const length = readUlebNumber(reader, MAX_COLUMN_VALUES); + decodeAssert(values.length + length <= count, "bool RLE has too many elements"); + for (let index = 0; index < length; index += 1) { + values.push(state); + } + state = !state; + } + return [values, bytes.subarray(reader.position)]; +} +export function decodeRleU8(bytes) { + return decodeAnyRle(bytes, readPostcardU8); +} +export function encodeRleU8(values) { + return encodeAnyRleLiteral(values, writePostcardU8); +} +export function decodeRleU32(bytes) { + return decodeAnyRle(bytes, readPostcardU32); +} +export function encodeRleU32(values) { + return encodeAnyRleLiteral(values, writePostcardU32); +} +export function decodeAnyRleU32(bytes) { + return decodeAnyRle(bytes, readPostcardU32); +} +export function encodeAnyRleU32(values) { + return encodeAnyRleLiteral(values, writePostcardU32); +} +export function takeAnyRleU32(bytes, count) { + return takeAnyRle(bytes, count, readPostcardU32); +} +export function decodeAnyRleU64(bytes) { + return decodeAnyRle(bytes, readPostcardU64); +} +export function encodeAnyRleU64(values) { + return encodeAnyRleLiteral(values, writePostcardU64); +} +export function decodeAnyRleUsize(bytes) { + return decodeAnyRleU64(bytes); +} +export function encodeAnyRleUsize(values) { + return encodeAnyRleU64(values); +} +export function takeAnyRleUsize(bytes, count) { + return takeAnyRle(bytes, count, readPostcardU64); +} +export function decodeAnyRleI32(bytes) { + return decodeAnyRle(bytes, readPostcardI32); +} +export function encodeAnyRleI32(values) { + return encodeAnyRleLiteral(values, writePostcardI32); +} +export function takeAnyRleI32(bytes, count) { + return takeAnyRle(bytes, count, readPostcardI32); +} +export function decodeDeltaRleU32(bytes) { + return decodeDeltaNumber(bytes, 0, U32_MAX); +} +export function encodeDeltaRleU32(values) { + return encodeDeltaNumber(values, assertU32); +} +export function decodeDeltaRleI32(bytes) { + return decodeDeltaNumber(bytes, I32_MIN, I32_MAX); +} +export function encodeDeltaRleI32(values) { + return encodeDeltaNumber(values, assertI32); +} +export function decodeDeltaRleUsize(bytes) { + return decodeDelta(bytes, 0n, U64_MAX, identity); +} +export function encodeDeltaRleUsize(values) { + return encodeDelta(values, assertUsize); +} +export function decodeDeltaRleIsize(bytes) { + return decodeDelta(bytes, I64_MIN, I64_MAX, identity); +} +export function encodeDeltaRleIsize(values) { + return encodeDelta(values, assertIsize); +} +export function decodeDeltaOfDeltaI64(bytes) { + const reader = new ByteReader(bytes); + const first = readPostcardOptionalI64(reader); + decodeAssert(reader.remaining >= 1, "invalid delta-of-delta bytes", reader.position); + const lastUsedBits = reader.readU8(); + const bitstream = reader.readRemaining(); + if (first === undefined) { + decodeAssert(lastUsedBits === 0 && bitstream.length === 0, "invalid empty delta-of-delta encoding"); + return []; + } + const bits = BitReader.forCompleteStream(bitstream, lastUsedBits); + const values = [first]; + let previous = first; + let delta = 0n; + // Number fast path: while every intermediate value stays a safe integer, + // accumulate with plain number arithmetic and box only the emitted element. + let previousNumber = 0; + let deltaNumber = 0; + let useNumber = first >= MIN_SAFE_BIGINT && first <= MAX_SAFE_BIGINT; + if (useNumber) { + previousNumber = Number(first); + } + while (bits.remaining > 0) { + const step = decodeDeltaOfDeltaValue(bits); + if (useNumber) { + if (typeof step === "number") { + const nextDelta = deltaNumber + step; + const next = previousNumber + nextDelta; + if (Number.isSafeInteger(nextDelta) && Number.isSafeInteger(next)) { + deltaNumber = nextDelta; + previousNumber = next; + values.push(BigInt(next)); + continue; + } + } + delta = BigInt(deltaNumber); + previous = BigInt(previousNumber); + useNumber = false; + } + const stepBigInt = typeof step === "number" ? BigInt(step) : step; + delta = checkedI64(delta + stepBigInt, "delta-of-delta delta"); + previous = checkedI64(previous + delta, "delta-of-delta value"); + values.push(previous); + } + return values; +} +export function takeDeltaOfDeltaI64(bytes, count) { + assertCount(count); + const reader = new ByteReader(bytes); + const first = readPostcardOptionalI64(reader); + decodeAssert(reader.remaining >= 1, "invalid delta-of-delta bytes", reader.position); + const lastUsedBits = reader.readU8(); + const bitstreamOffset = reader.position; + if (first === undefined) { + decodeAssert(count === 0, "delta-of-delta has too few elements"); + decodeAssert(lastUsedBits === 0, "invalid empty delta-of-delta encoding"); + return [[], bytes.subarray(bitstreamOffset)]; + } + decodeAssert(count > 0, "delta-of-delta has too many elements"); + const bits = BitReader.forPrefix(bytes.subarray(bitstreamOffset)); + const values = [first]; + let previous = first; + let delta = 0n; + // Number fast path, mirroring decodeDeltaOfDeltaI64. + let previousNumber = 0; + let deltaNumber = 0; + let useNumber = first >= MIN_SAFE_BIGINT && first <= MAX_SAFE_BIGINT; + if (useNumber) { + previousNumber = Number(first); + } + while (values.length < count) { + const step = decodeDeltaOfDeltaValue(bits); + if (useNumber) { + if (typeof step === "number") { + const nextDelta = deltaNumber + step; + const next = previousNumber + nextDelta; + if (Number.isSafeInteger(nextDelta) && Number.isSafeInteger(next)) { + deltaNumber = nextDelta; + previousNumber = next; + values.push(BigInt(next)); + continue; + } + } + delta = BigInt(deltaNumber); + previous = BigInt(previousNumber); + useNumber = false; + } + const stepBigInt = typeof step === "number" ? BigInt(step) : step; + delta = checkedI64(delta + stepBigInt, "delta-of-delta delta"); + previous = checkedI64(previous + delta, "delta-of-delta value"); + values.push(previous); + } + if (count === 1) { + decodeAssert(lastUsedBits === 0, "invalid single-value delta-of-delta encoding"); + } + else { + const expectedLastUsed = bits.position % 8 || 8; + decodeAssert(lastUsedBits === expectedLastUsed, "delta-of-delta last-used-bits mismatch"); + } + return [values, bytes.subarray(bitstreamOffset + Math.ceil(bits.position / 8))]; +} +export function encodeDeltaOfDeltaI64(values) { + const writer = new ByteWriter(); + if (values.length === 0) { + writePostcardOptionalI64(writer, undefined); + writer.writeU8(0); + return writer.toUint8Array(); + } + for (const value of values) { + assertBigIntRange(value, I64_MIN, I64_MAX, "i64"); + } + writePostcardOptionalI64(writer, values[0]); + if (values.length === 1) { + writer.writeU8(0); + return writer.toUint8Array(); + } + const bits = new BitWriter(); + let previousDelta = 0n; + // Number fast path: differences of safe integers stay exact while the + // running delta and delta-of-delta remain safe integers. + let previousDeltaNumber = 0; + let useNumber = true; + for (let index = 1; index < values.length; index += 1) { + const current = values[index]; + const prior = values[index - 1]; + if (useNumber) { + if (current >= MIN_SAFE_BIGINT && + current <= MAX_SAFE_BIGINT && + prior >= MIN_SAFE_BIGINT && + prior <= MAX_SAFE_BIGINT) { + const deltaNumber = Number(current) - Number(prior); + const difference = deltaNumber - previousDeltaNumber; + if (Number.isSafeInteger(deltaNumber) && Number.isSafeInteger(difference)) { + previousDeltaNumber = deltaNumber; + encodeDeltaOfDeltaValueNumber(bits, difference); + continue; + } + } + previousDelta = BigInt(previousDeltaNumber); + useNumber = false; + } + const delta = checkedI64(current - prior, "delta-of-delta delta"); + const deltaOfDelta = checkedI64(delta - previousDelta, "delta-of-delta difference"); + previousDelta = delta; + encodeDeltaOfDeltaValue(bits, deltaOfDelta); + } + const encoded = bits.finish(); + writer.writeU8(encoded.lastUsedBits); + writer.writeBytes(encoded.bytes); + return writer.toUint8Array(); +} +function readColumnarVec(reader) { + const count = readUlebNumber(reader, MAX_COLUMN_VALUES); + const columns = []; + for (let index = 0; index < count; index += 1) { + columns.push(reader.readBytes(readUlebNumber(reader, 2147483647))); + } + return columns; +} +function decodeAnyRle(bytes, readValue) { + const reader = new ByteReader(bytes); + const values = []; + while (reader.remaining > 0) { + readAnyRleSegment(reader, values, MAX_COLUMN_VALUES, readValue); + } + return values; +} +function takeAnyRle(bytes, count, readValue) { + assertCount(count); + const reader = new ByteReader(bytes); + const values = []; + while (values.length < count) { + decodeAssert(reader.remaining > 0, "AnyRle has too few elements", reader.position); + readAnyRleSegment(reader, values, count, readValue); + } + return [values, bytes.subarray(reader.position)]; +} +function readAnyRleSegment(reader, values, limit, readValue) { + const signedLength = readPostcardI64(reader); + decodeAssert(signedLength !== 0n, "AnyRle segment has zero length", reader.position); + const lengthBigInt = signedLength < 0n ? -signedLength : signedLength; + decodeAssert(lengthBigInt <= BigInt(MAX_COLUMN_VALUES), "AnyRle segment is too long"); + const length = Number(lengthBigInt); + decodeAssert(values.length + length <= limit, "AnyRle has too many elements"); + if (signedLength > 0n) { + const value = readValue(reader); + for (let index = 0; index < length; index += 1) { + values.push(value); + } + return; + } + for (let index = 0; index < length; index += 1) { + values.push(readValue(reader)); + } +} +function encodeAnyRleLiteral(values, writeValue) { + const writer = new ByteWriter(); + if (values.length === 0) { + return writer.toUint8Array(); + } + if (values.length > MAX_COLUMN_VALUES) { + throw new LoroEncodeError(`AnyRle input is too long: ${values.length}`); + } + writePostcardI64(writer, -BigInt(values.length)); + for (const value of values) { + writeValue(writer, value); + } + return writer.toUint8Array(); +} +function encodeDelta(values, toBigInt) { + const writer = new ByteWriter(); + if (values.length === 0) { + return writer.toUint8Array(); + } + if (values.length > MAX_COLUMN_VALUES) { + throw new LoroEncodeError(`AnyRle input is too long: ${values.length}`); + } + writePostcardI64(writer, -BigInt(values.length)); + let previous = 0n; + // Number fast path: safe-integer values encode through the number i128 + // writer, avoiding per-element BigInt subtraction and comparison. + let previousNumber = 0; + let useNumber = true; + for (const input of values) { + const value = toBigInt(input); + if (useNumber) { + if (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) { + const valueNumber = Number(value); + const deltaNumber = valueNumber - previousNumber; + if (Number.isSafeInteger(deltaNumber)) { + writePostcardI128Number(writer, deltaNumber); + previousNumber = valueNumber; + continue; + } + } + previous = BigInt(previousNumber); + useNumber = false; + } + const delta = value - previous; + assertBigIntRange(delta, I128_MIN, I128_MAX, "delta RLE delta"); + writePostcardI128(writer, delta); + previous = value; + } + return writer.toUint8Array(); +} +function decodeDelta(bytes, min, max, fromBigInt) { + const reader = new ByteReader(bytes); + const values = []; + let value = 0n; + // Number fast path: accumulate with plain number arithmetic while the + // running value stays a safe integer. A safe integer always compares + // exactly against these bounds: any bound outside the safe range is + // replaced by the nearest safe-integer limit, which accepts/rejects the + // same values as the BigInt comparison. + let valueNumber = 0; + let useNumber = true; + const minNumber = min < MIN_SAFE_BIGINT ? Number.MIN_SAFE_INTEGER : Number(min); + const maxNumber = max > MAX_SAFE_BIGINT ? Number.MAX_SAFE_INTEGER : Number(max); + const append = (delta) => { + if (useNumber) { + if (typeof delta === "number") { + const next = valueNumber + delta; + if (Number.isSafeInteger(next)) { + valueNumber = next; + decodeAssert(next >= minNumber && next <= maxNumber, "delta RLE value is out of range"); + values.push(fromBigInt(BigInt(next))); + return; + } + } + value = BigInt(valueNumber); + useNumber = false; + } + const deltaBigInt = typeof delta === "number" ? BigInt(delta) : delta; + value += deltaBigInt; + decodeAssert(value >= min && value <= max, "delta RLE value is out of range"); + values.push(fromBigInt(value)); + }; + while (reader.remaining > 0) { + const signedLength = readPostcardI64(reader); + decodeAssert(signedLength !== 0n, "AnyRle segment has zero length", reader.position); + const lengthBigInt = signedLength < 0n ? -signedLength : signedLength; + decodeAssert(lengthBigInt <= BigInt(MAX_COLUMN_VALUES), "AnyRle segment is too long"); + const length = Number(lengthBigInt); + decodeAssert(values.length + length <= MAX_COLUMN_VALUES, "AnyRle has too many elements"); + if (signedLength > 0n) { + const delta = readPostcardI128NumberOrBigInt(reader); + for (let index = 0; index < length; index += 1) { + append(delta); + } + } + else { + for (let index = 0; index < length; index += 1) { + append(readPostcardI128NumberOrBigInt(reader)); + } + } + } + return values; +} +function encodeDeltaNumber(values, validate) { + const writer = new ByteWriter(); + if (values.length === 0) { + return writer.toUint8Array(); + } + if (values.length > MAX_COLUMN_VALUES) { + throw new LoroEncodeError(`AnyRle input is too long: ${values.length}`); + } + writePostcardI64(writer, -BigInt(values.length)); + let previous = 0; + for (const input of values) { + const value = validate(input); + writePostcardI128Number(writer, value - previous); + previous = value; + } + return writer.toUint8Array(); +} +function decodeDeltaNumber(bytes, min, max) { + const reader = new ByteReader(bytes); + const values = []; + let value = 0; + const append = (delta) => { + value += delta; + decodeAssert(value >= min && value <= max, "delta RLE value is out of range"); + values.push(value); + }; + while (reader.remaining > 0) { + const signedLength = readPostcardI64(reader); + decodeAssert(signedLength !== 0n, "AnyRle segment has zero length", reader.position); + const lengthBigInt = signedLength < 0n ? -signedLength : signedLength; + decodeAssert(lengthBigInt <= BigInt(MAX_COLUMN_VALUES), "AnyRle segment is too long"); + const length = Number(lengthBigInt); + decodeAssert(values.length + length <= MAX_COLUMN_VALUES, "AnyRle has too many elements"); + if (signedLength > 0n) { + const delta = readPostcardI128Number(reader); + for (let index = 0; index < length; index += 1) { + append(delta); + } + } + else { + for (let index = 0; index < length; index += 1) { + append(readPostcardI128Number(reader)); + } + } + } + return values; +} +function readPostcardU8(reader) { + return reader.readU8(); +} +function writePostcardU8(writer, value) { + writer.writeU8(value); +} +function readPostcardU32(reader) { + return readUlebNumber(reader, 4294967295); +} +function writePostcardU32(writer, value) { + if (!Number.isSafeInteger(value) || value < 0 || value > 4294967295) { + throw new LoroEncodeError(`u32 is out of range: ${value}`); + } + writeUleb128(writer, value); +} +function readPostcardU64(reader) { + return readUleb128(reader, U64_MAX); +} +function writePostcardU64(writer, value) { + writeUleb128(writer, value); +} +function readPostcardI32(reader) { + const encoded = readUlebNumber(reader, 4294967295); + // ZigZag decode; every u32-encoded value lands in the i32 range. + const value = (encoded >>> 1) ^ -(encoded & 1); + if (value < I32_MIN || value > I32_MAX) { + throw new LoroDecodeError("postcard signed integer is out of range"); + } + return value; +} +function writePostcardI32(writer, value) { + if (!Number.isSafeInteger(value) || value < I32_MIN || value > I32_MAX) { + throw new LoroEncodeError(`i32 is out of range: ${value}`); + } + writeUleb128(writer, ((value << 1) ^ (value >> 31)) >>> 0); +} +function readPostcardI64(reader) { + const encoded = readUleb128(reader, U64_MAX); + const value = (encoded >> 1n) ^ -(encoded & 1n); + if (value < I64_MIN || value > I64_MAX) { + throw new LoroDecodeError("postcard signed integer is out of range"); + } + return value; +} +function writePostcardI64(writer, value) { + if (value < I64_MIN || value > I64_MAX) { + throw new LoroEncodeError(`i64 is out of range: ${value}`); + } + writeUleb128(writer, (value << 1n) ^ (value >> 63n)); +} +function readPostcardOptionalI64(reader) { + const tag = readPostcardU64(reader); + if (tag === 0n) { + return undefined; + } + if (tag === 1n) { + return readPostcardI64(reader); + } + throw new LoroDecodeError("invalid postcard Option tag", reader.position); +} +function writePostcardOptionalI64(writer, value) { + writePostcardU64(writer, value === undefined ? 0n : 1n); + if (value !== undefined) { + writePostcardI64(writer, value); + } +} +function writePostcardI128(writer, value) { + assertBigIntRange(value, I128_MIN, I128_MAX, "i128"); + const encoded = value >= 0n ? value << 1n : (-value << 1n) - 1n; + writeVarintU128(writer, encoded); +} +function readPostcardI128Number(reader) { + const start = reader.position; + let encoded = 0; + let multiplier = 1; + for (let index = 0; index < 19; index += 1) { + const byte = reader.readU8(); + const payload = byte & 0x7f; + if (payload !== 0 && multiplier > Number.MAX_SAFE_INTEGER / payload) { + throw new LoroDecodeError("i128 varint exceeds safe integer range", start); + } + encoded += payload * multiplier; + if (!Number.isSafeInteger(encoded)) { + throw new LoroDecodeError("i128 varint exceeds safe integer range", start); + } + if ((byte & 0x80) === 0) { + if (encoded < 2147483648) { + // Smi ZigZag decode; identical to the division-based formula below. + return (encoded >>> 1) ^ -(encoded & 1); + } + return encoded % 2 === 0 ? encoded / 2 : -(encoded + 1) / 2; + } + multiplier *= 128; + } + throw new LoroDecodeError("u128 varint overflow", start); +} +function writePostcardI128Number(writer, value) { + if (Number.isSafeInteger(value) && value > -2147483648 && value < 2147483648) { + // Smi-only ZigZag encode and varint emission for 31-bit values. + let encoded = ((value << 1) ^ (value >> 31)) >>> 0; + do { + let byte = encoded & 0x7f; + encoded >>>= 7; + if (encoded !== 0) { + byte |= 0x80; + } + writer.writeU8(byte); + } while (encoded !== 0); + return; + } + const encoded = value >= 0 ? value * 2 : -value * 2 - 1; + let remaining = encoded; + do { + let byte = remaining % 128; + remaining = Math.floor(remaining / 128); + if (remaining !== 0) { + byte |= 0x80; + } + writer.writeU8(byte); + } while (remaining !== 0); +} +// Reads a postcard i128, keeping the whole decode in number arithmetic while +// the encoded varint stays a safe integer and falling back to BigInt only for +// larger (still valid u128) inputs. +function readPostcardI128NumberOrBigInt(reader) { + const start = reader.position; + let encoded = 0; + let multiplier = 1; + let encodedBigInt = 0n; + let useBigInt = false; + for (let index = 0; index < 19; index += 1) { + const byte = reader.readU8(); + const payload = byte & 0x7f; + if (index === 18 && payload > 3) { + throw new LoroDecodeError("u128 varint overflow", start); + } + if (useBigInt) { + encodedBigInt |= BigInt(payload) << BigInt(index * 7); + } + else if (payload !== 0) { + if (multiplier <= Number.MAX_SAFE_INTEGER / payload) { + const sum = encoded + payload * multiplier; + if (Number.isSafeInteger(sum)) { + encoded = sum; + } + else { + useBigInt = true; + encodedBigInt = BigInt(encoded) | (BigInt(payload) << BigInt(index * 7)); + } + } + else { + useBigInt = true; + encodedBigInt = BigInt(encoded) | (BigInt(payload) << BigInt(index * 7)); + } + } + if ((byte & 0x80) === 0) { + if (!useBigInt) { + if (encoded < 2147483648) { + return (encoded >>> 1) ^ -(encoded & 1); + } + return encoded % 2 === 0 ? encoded / 2 : -(encoded + 1) / 2; + } + const value = (encodedBigInt >> 1n) ^ -(encodedBigInt & 1n); + decodeAssert(value >= I128_MIN && value <= I128_MAX, "postcard i128 is out of range"); + return value; + } + multiplier *= 128; + } + throw new LoroDecodeError("u128 varint overflow", start); +} +function writeVarintU128(writer, input) { + assertBigIntRange(input, 0n, U128_MAX, "u128"); + let value = input; + do { + let byte = Number(value & 0x7fn); + value >>= 7n; + if (value !== 0n) { + byte |= 0x80; + } + writer.writeU8(byte); + } while (value !== 0n); +} +class BitReader { + #bytes; + #totalBits; + #bytePosition = 0; + // Pending bits read from the stream but not yet consumed, right-aligned. + // The invariant #bitCount <= 7 holds between read calls, so every + // intermediate below stays a positive 31-bit value and bitwise ops are + // exact Smi operations. + #bitBuffer = 0; + #bitCount = 0; + constructor(bytes, totalBits) { + this.#bytes = bytes; + this.#totalBits = totalBits; + } + static forCompleteStream(bytes, lastUsedBits) { + decodeAssert(lastUsedBits >= 0 && lastUsedBits <= 8, "invalid last-used-bits value"); + if (lastUsedBits === 0) { + decodeAssert(bytes.length === 0, "unexpected delta-of-delta bitstream bytes"); + return new BitReader(bytes, 0); + } + decodeAssert(bytes.length > 0, "missing delta-of-delta bitstream bytes"); + return new BitReader(bytes, (bytes.length - 1) * 8 + lastUsedBits); + } + static forPrefix(bytes) { + return new BitReader(bytes, bytes.length * 8); + } + get position() { + return this.#bytePosition * 8 - this.#bitCount; + } + get remaining() { + return this.#totalBits - this.position; + } + // Reads up to 24 bits (the refill loop would exceed 31-bit intermediate + // values above that) and returns them as a number. Bits are consumed + // MSB-first, matching the original bit-at-a-time implementation. + readBitsNumber(count) { + decodeAssert(Number.isSafeInteger(count) && count >= 0 && count <= 24, "invalid bit width"); + decodeAssert(this.remaining >= count, "unexpected end of delta-of-delta bitstream"); + while (this.#bitCount < count) { + this.#bitBuffer = (this.#bitBuffer << 8) | this.#bytes[this.#bytePosition]; + this.#bytePosition += 1; + this.#bitCount += 8; + } + const shift = this.#bitCount - count; + const value = this.#bitBuffer >>> shift; + this.#bitBuffer &= (1 << shift) - 1; + this.#bitCount -= count; + return value; + } + readBitsBigInt(count) { + decodeAssert(Number.isSafeInteger(count) && count >= 0 && count <= 64, "invalid bit width"); + decodeAssert(this.remaining >= count, "unexpected end of delta-of-delta bitstream"); + let value = 0n; + let remaining = count; + while (remaining > 0) { + const step = Math.min(remaining, 24); + value = (value << BigInt(step)) | BigInt(this.readBitsNumber(step)); + remaining -= step; + } + return value; + } +} +class BitWriter { + #bytes = []; + // Same right-aligned pending-bits representation as BitReader, with the + // invariant #bitCount <= 7 between write calls. + #bitBuffer = 0; + #bitCount = 0; + // Writes the low `count` bits of value, MSB first. Like the original + // bit-at-a-time loop, bits of value above `count` are ignored. + writeBitsNumber(value, count) { + encodeAssert(Number.isSafeInteger(count) && count >= 0 && count <= 24, "invalid bit width"); + this.#bitBuffer = (this.#bitBuffer << count) | (value & ((1 << count) - 1)); + this.#bitCount += count; + while (this.#bitCount >= 8) { + const shift = this.#bitCount - 8; + this.#bytes.push(this.#bitBuffer >>> shift); + this.#bitBuffer &= (1 << shift) - 1; + this.#bitCount -= 8; + } + } + writeBitsBigInt(value, count) { + encodeAssert(count >= 0 && count <= 64, "invalid bit width"); + let remaining = count; + while (remaining > 0) { + const step = Math.min(remaining, 24); + remaining -= step; + this.writeBitsNumber(Number((value >> BigInt(remaining)) & 0xffffffn), step); + } + } + finish() { + if (this.#bitCount === 0) { + return { bytes: Uint8Array.from(this.#bytes), lastUsedBits: 8 }; + } + this.#bytes.push(this.#bitBuffer << (8 - this.#bitCount)); + return { bytes: Uint8Array.from(this.#bytes), lastUsedBits: this.#bitCount }; + } +} +function decodeDeltaOfDeltaValue(reader) { + if (reader.readBitsNumber(1) === 0) { + return 0; + } + if (reader.readBitsNumber(1) === 0) { + return reader.readBitsNumber(7) - 63; + } + if (reader.readBitsNumber(1) === 0) { + return reader.readBitsNumber(9) - 255; + } + if (reader.readBitsNumber(1) === 0) { + return reader.readBitsNumber(12) - 2047; + } + if (reader.readBitsNumber(1) === 0) { + return reader.readBitsNumber(21) - 1048575; + } + return BigInt.asIntN(64, reader.readBitsBigInt(64)); +} +function encodeDeltaOfDeltaValue(writer, value) { + if (value === 0n) { + writer.writeBitsNumber(0, 1); + } + else if (value >= -63n && value <= 64n) { + writer.writeBitsNumber(0b10, 2); + writer.writeBitsNumber(Number(value + 63n), 7); + } + else if (value >= -255n && value <= 256n) { + writer.writeBitsNumber(0b110, 3); + writer.writeBitsNumber(Number(value + 255n), 9); + } + else if (value >= -2047n && value <= 2048n) { + writer.writeBitsNumber(0b1110, 4); + writer.writeBitsNumber(Number(value + 2047n), 12); + } + else if (value >= -1048575n && value <= 1048576n) { + writer.writeBitsNumber(0b11110, 5); + writer.writeBitsNumber(Number(value + 1048575n), 21); + } + else { + writer.writeBitsNumber(0b11111, 5); + writer.writeBitsBigInt(BigInt.asUintN(64, value), 64); + } +} +function encodeDeltaOfDeltaValueNumber(writer, value) { + if (value === 0) { + writer.writeBitsNumber(0, 1); + } + else if (value >= -63 && value <= 64) { + writer.writeBitsNumber(0b10, 2); + writer.writeBitsNumber(value + 63, 7); + } + else if (value >= -255 && value <= 256) { + writer.writeBitsNumber(0b110, 3); + writer.writeBitsNumber(value + 255, 9); + } + else if (value >= -2047 && value <= 2048) { + writer.writeBitsNumber(0b1110, 4); + writer.writeBitsNumber(value + 2047, 12); + } + else if (value >= -1048575 && value <= 1048576) { + writer.writeBitsNumber(0b11110, 5); + writer.writeBitsNumber(value + 1048575, 21); + } + else { + writer.writeBitsNumber(0b11111, 5); + writer.writeBitsBigInt(BigInt.asUintN(64, BigInt(value)), 64); + } +} +function checkedI64(value, label) { + decodeAssert(value >= I64_MIN && value <= I64_MAX, `${label} is out of range`); + return value; +} +function assertDecodedLength(current, extra, label) { + decodeAssert(current + extra <= MAX_COLUMN_VALUES, `${label} output is too long`); +} +function assertCount(count) { + if (!Number.isSafeInteger(count) || count < 0 || count > MAX_COLUMN_VALUES) { + throw new LoroDecodeError(`invalid requested column value count: ${count}`); + } +} +function assertU32(value) { + if (!Number.isSafeInteger(value) || value < 0 || value > 4294967295) { + throw new LoroEncodeError(`u32 is out of range: ${value}`); + } + return value; +} +function assertI32(value) { + if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { + throw new LoroEncodeError(`i32 is out of range: ${value}`); + } + return value; +} +function assertUsize(value) { + assertBigIntRange(value, 0n, U64_MAX, "usize"); + return value; +} +function assertIsize(value) { + assertBigIntRange(value, I64_MIN, I64_MAX, "isize"); + return value; +} +function identity(value) { + return value; +} +function assertBigIntRange(value, min, max, label) { + if (value < min || value > max) { + throw new LoroEncodeError(`${label} is out of range: ${value}`); + } +} diff --git a/loro-js/src/codec/sstable.js b/loro-js/src/codec/sstable.js new file mode 100644 index 000000000..3194a803f --- /dev/null +++ b/loro-js/src/codec/sstable.js @@ -0,0 +1,261 @@ +import { ByteReader, ByteWriter, bytesEqual, compareBytes } from "./bytes"; +import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; +import { decodeLz4Frame, encodeLz4FrameRaw } from "./lz4"; +import { LORO_XXHASH_SEED, xxhash32 } from "./xxhash32"; +const SSTABLE_MAGIC = Uint8Array.of(0x4c, 0x4f, 0x52, 0x4f); +const SSTABLE_SCHEMA = 0; +const MAX_BLOCK_COUNT = 10000000; +export function decodeSstable(bytes, options = {}) { + if (bytes.length === 0) { + return []; + } + decodeAssert(bytes.length >= 17, "SSTable is too short", 0); + decodeAssert(bytesEqual(bytes.subarray(0, 4), SSTABLE_MAGIC), "invalid SSTable magic", 0); + decodeAssert(bytes[4] === SSTABLE_SCHEMA, "unsupported SSTable schema", 4); + const footer = new ByteReader(bytes, bytes.length - 4, 4); + const metadataOffset = footer.readU32LE(); + decodeAssert(metadataOffset >= 5 && metadataOffset < bytes.length - 4, "invalid SSTable metadata offset", bytes.length - 4); + const metadataBytes = bytes.subarray(metadataOffset, bytes.length - 4); + const metadata = decodeMetadata(metadataBytes, options.checkChecksum !== false); + const entries = []; + for (let index = 0; index < metadata.length; index += 1) { + const current = metadata[index]; + const end = index + 1 < metadata.length ? metadata[index + 1].offset : metadataOffset; + decodeAssert(current.offset >= 5, "invalid SSTable block offset", current.offset); + decodeAssert(end > current.offset && end <= metadataOffset, "invalid SSTable block range", current.offset); + const stored = bytes.subarray(current.offset, end); + decodeAssert(stored.length >= 4, "SSTable block lacks checksum", current.offset); + const payload = stored.subarray(0, stored.length - 4); + const checksum = new ByteReader(stored, stored.length - 4, 4).readU32LE(); + if (options.checkChecksum !== false) { + decodeAssert(checksum === xxhash32(payload, LORO_XXHASH_SEED), "SSTable block checksum mismatch", end - 4); + } + const decoded = current.compression === 0 ? payload : decodeLz4Frame(payload, options); + if (current.large) { + entries.push({ key: current.firstKey, value: decoded }); + continue; + } + const blockEntries = decodeNormalBlock(decoded, current); + entries.push(...blockEntries); + } + validateEntryOrder(entries); + return entries; +} +export function encodeSstable(input, options = {}) { + if (input.length === 0) { + return new Uint8Array(); + } + const blockSize = options.blockSize ?? 4096; + const compression = options.compression ?? "auto"; + if (!Number.isSafeInteger(blockSize) || blockSize <= 0 || blockSize > 0xffff) { + throw new LoroEncodeError(`invalid SSTable block size ${blockSize}`); + } + // Sorting only needs to reorder references; keys and values are copied into + // the output while encoding and never retained past the returned bytes. + const entries = input.slice(); + entries.sort((left, right) => compareBytes(left.key, right.key)); + validateEntriesForEncoding(entries); + const blocks = []; + for (let index = 0; index < entries.length;) { + const first = entries[index]; + if (first.value.length > blockSize || first.value.length > 0xffff) { + blocks.push(encodeStoredBlock(first.value, true, first.key, undefined, compression)); + index += 1; + continue; + } + const data = new ByteWriter(); + const offsets = [0]; + const firstKey = first.key; + let lastKey = firstKey; + data.writeBytes(first.value); + index += 1; + while (index < entries.length) { + const next = entries[index]; + if (next.value.length > blockSize || next.value.length > 0xffff) { + break; + } + const common = commonPrefixLength(firstKey, next.key); + const suffixLength = next.key.length - common; + if (suffixLength > 0xffff || data.length > 0xffff) { + break; + } + const added = 3 + suffixLength + next.value.length; + const prospective = data.length + added + (offsets.length + 1) * 2 + 2; + if (prospective > blockSize) { + break; + } + offsets.push(data.length); + data.writeU8(common); + data.writeU16LE(suffixLength); + data.writeBytes(next.key.subarray(common)); + data.writeBytes(next.value); + lastKey = next.key; + index += 1; + } + const body = new ByteWriter(data.length + offsets.length * 2 + 2); + body.writeBytes(data.toUint8Array()); + for (const offset of offsets) { + body.writeU16LE(offset); + } + body.writeU16LE(offsets.length); + blocks.push(encodeStoredBlock(body.toUint8Array(), false, firstKey, lastKey, compression)); + } + return encodeTable(blocks); +} +function decodeMetadata(bytes, checkChecksum) { + decodeAssert(bytes.length >= 8, "SSTable metadata is too short"); + const reader = new ByteReader(bytes); + const count = reader.readU32LE(); + decodeAssert(count > 0 && count <= MAX_BLOCK_COUNT, "invalid SSTable block count", 0); + if (checkChecksum) { + const stored = new ByteReader(bytes, bytes.length - 4, 4).readU32LE(); + const expected = xxhash32(bytes.subarray(4, bytes.length - 4), LORO_XXHASH_SEED); + decodeAssert(stored === expected, "SSTable metadata checksum mismatch", bytes.length - 4); + } + const metadata = []; + for (let index = 0; index < count; index += 1) { + const offset = reader.readU32LE(); + const firstKey = reader.readBytes(reader.readU16LE()); + const flags = reader.readU8(); + const large = (flags & 0x80) !== 0; + const compression = flags & 0x7f; + decodeAssert(compression === 0 || compression === 1, "invalid SSTable compression tag", reader.position - 1); + const lastKey = large ? undefined : reader.readBytes(reader.readU16LE()); + metadata.push({ offset, large, compression, firstKey, lastKey }); + } + reader.readU32LE(); + reader.assertEnd("trailing SSTable metadata bytes"); + for (let index = 0; index < metadata.length; index += 1) { + const current = metadata[index]; + decodeAssert(current.firstKey.length > 0, "empty SSTable metadata key"); + if (index > 0) { + decodeAssert(current.offset > metadata[index - 1].offset, "SSTable block offsets are not strictly increasing"); + decodeAssert(compareBytes(metadata[index - 1].firstKey, current.firstKey) < 0, "SSTable metadata keys are not strictly increasing"); + } + } + return metadata; +} +function decodeNormalBlock(bytes, metadata) { + decodeAssert(bytes.length >= 2, "normal SSTable block is too short"); + const count = new ByteReader(bytes, bytes.length - 2, 2).readU16LE(); + decodeAssert(count > 0, "normal SSTable block is empty"); + const offsetsLength = count * 2; + const dataEnd = bytes.length - offsetsLength - 2; + decodeAssert(dataEnd >= 0, "normal SSTable offsets exceed block length"); + const offsetsReader = new ByteReader(bytes, dataEnd, offsetsLength); + const offsets = []; + for (let index = 0; index < count; index += 1) { + offsets.push(offsetsReader.readU16LE()); + } + decodeAssert(offsets[0] === 0, "first normal SSTable offset is not zero"); + const entries = []; + for (let index = 0; index < count; index += 1) { + const start = offsets[index]; + const end = index + 1 < count ? offsets[index + 1] : dataEnd; + decodeAssert(start <= end && end <= dataEnd, "invalid normal SSTable entry range"); + const entry = bytes.subarray(start, end); + if (index === 0) { + entries.push({ key: metadata.firstKey, value: entry }); + continue; + } + const reader = new ByteReader(entry); + const common = reader.readU8(); + decodeAssert(common <= metadata.firstKey.length, "invalid SSTable key prefix length"); + const suffix = reader.readBytes(reader.readU16LE()); + const key = new Uint8Array(common + suffix.length); + key.set(metadata.firstKey.subarray(0, common)); + key.set(suffix, common); + entries.push({ key, value: reader.readRemaining() }); + } + validateEntryOrder(entries); + const last = entries[entries.length - 1]; + decodeAssert(metadata.lastKey !== undefined && bytesEqual(last.key, metadata.lastKey), "SSTable metadata last key mismatch"); + return entries; +} +function encodeStoredBlock(raw, large, firstKey, lastKey, compression) { + const lz4 = compression === "none" ? undefined : encodeLz4FrameRaw(raw); + const useLz4 = compression === "lz4" || (compression === "auto" && lz4.length <= raw.length); + const payload = useLz4 ? lz4 : raw; + const writer = new ByteWriter(payload.length + 4); + writer.writeBytes(payload); + writer.writeU32LE(xxhash32(payload, LORO_XXHASH_SEED)); + return { + bytes: writer.toUint8Array(), + large, + compression: useLz4 ? 1 : 0, + firstKey, + lastKey, + }; +} +function encodeTable(blocks) { + encodeAssert(blocks.length > 0 && blocks.length <= MAX_BLOCK_COUNT, "invalid SSTable block count"); + let offset = 5; + const metadata = []; + for (const block of blocks) { + metadata.push({ + offset, + large: block.large, + compression: block.compression, + firstKey: block.firstKey, + lastKey: block.lastKey, + }); + offset += block.bytes.length; + encodeAssert(offset <= 4294967295, "SSTable exceeds u32 offsets"); + } + const metadataOffset = offset; + const metadataWriter = new ByteWriter(); + metadataWriter.writeU32LE(metadata.length); + for (const item of metadata) { + encodeAssert(item.firstKey.length <= 0xffff, "SSTable key exceeds u16 length"); + metadataWriter.writeU32LE(item.offset); + metadataWriter.writeU16LE(item.firstKey.length); + metadataWriter.writeBytes(item.firstKey); + metadataWriter.writeU8((item.large ? 0x80 : 0) | item.compression); + if (!item.large) { + encodeAssert(item.lastKey !== undefined, "normal SSTable block lacks last key"); + encodeAssert(item.lastKey.length <= 0xffff, "SSTable key exceeds u16 length"); + metadataWriter.writeU16LE(item.lastKey.length); + metadataWriter.writeBytes(item.lastKey); + } + } + const metadataWithoutChecksum = metadataWriter.toUint8Array(); + metadataWriter.writeU32LE(xxhash32(metadataWithoutChecksum.subarray(4), LORO_XXHASH_SEED)); + const writer = new ByteWriter(); + writer.writeBytes(SSTABLE_MAGIC); + writer.writeU8(SSTABLE_SCHEMA); + for (const block of blocks) { + writer.writeBytes(block.bytes); + } + writer.writeBytes(metadataWriter.toUint8Array()); + writer.writeU32LE(metadataOffset); + return writer.toUint8Array(); +} +function commonPrefixLength(a, b) { + const length = Math.min(255, a.length, b.length); + let index = 0; + while (index < length && a[index] === b[index]) { + index += 1; + } + return index; +} +function validateEntriesForEncoding(entries) { + for (let index = 0; index < entries.length; index += 1) { + const current = entries[index]; + if (current.key.length === 0) { + throw new LoroEncodeError("SSTable keys cannot be empty"); + } + if (current.key.length > 0xffff) { + throw new LoroEncodeError("SSTable key exceeds u16 length"); + } + if (index > 0 && compareBytes(entries[index - 1].key, current.key) >= 0) { + throw new LoroEncodeError("SSTable keys must be unique"); + } + } +} +function validateEntryOrder(entries) { + for (let index = 1; index < entries.length; index += 1) { + if (compareBytes(entries[index - 1].key, entries[index].key) >= 0) { + throw new LoroDecodeError("SSTable keys are not strictly increasing"); + } + } +} diff --git a/loro-js/src/codec/state-snapshot.js b/loro-js/src/codec/state-snapshot.js new file mode 100644 index 000000000..6e82072c4 --- /dev/null +++ b/loro-js/src/codec/state-snapshot.js @@ -0,0 +1,731 @@ +import { ByteReader, ByteWriter, bytesEqual, compareBytes } from "./bytes"; +import { containerTypeFromRawByte, containerTypeToRawByte, decodeContainerId, encodeContainerId, readPostcardOptionalContainerId, writePostcardOptionalContainerId, } from "./container-id"; +import { decodeAssert, encodeAssert } from "./errors"; +import { U64_MAX, readUleb128, readUlebNumber, writeUleb128 } from "./leb128"; +import { PostcardReader, PostcardWriter } from "./postcard"; +import { decodePositionArena, encodePositionArena } from "./position-arena"; +import { decodeBoolRle, decodeColumnarVecMaybeWrapped, decodeDeltaRleI32, decodeDeltaRleU32, decodeDeltaRleUsize, encodeBoolRle, encodeColumnarVec, encodeColumnarVecWrapped, encodeDeltaRleI32, encodeDeltaRleU32, encodeDeltaRleUsize, takeColumnarVec, } from "./serde-columnar"; +import { decodeSstable, encodeSstable, } from "./sstable"; +import { ContainerType, } from "./types"; +import { readPostcardValue, readPostcardValueMap, readPostcardValues, writePostcardValue, writePostcardValueMap, writePostcardValues, } from "./value"; +import { decodePostcardFrontiers, encodePostcardFrontiers } from "./version"; +const MAX_STATE_ITEMS = 10000000; +const EMPTY_STATE_SENTINEL = Uint8Array.of(0x45); +const FRONTIERS_KEY = Uint8Array.of(0x66, 0x72); +const textEncoder = new TextEncoder(); +export function decodeMapStateSnapshot(bytes) { + const reader = new PostcardReader(bytes); + const values = readPostcardValueMap(reader); + const deletedKeys = readStringVector(reader, "map deleted keys"); + const peers = readPeerTable(reader.input); + const keys = collectSortedMapKeysForDecode(values, deletedKeys); + const peerCount = BigInt(peers.length); + const metadata = []; + for (const key of keys) { + const peerIndex = readUleb128(reader.input, U64_MAX); + const lamport = readUleb128(reader.input, U64_MAX); + assertDecodedPeerIndex(peerIndex, peerCount, "map metadata"); + metadata.push({ key, peerIndex, lamport }); + } + reader.assertEnd(); + sortByUtf8Key(values, (entry) => entry[0]); + sortByUtf8Key(deletedKeys, (key) => key); + return { kind: ContainerType.Map, values, deletedKeys, peers, metadata }; +} +export function encodeMapStateSnapshot(state) { + const values = [...state.values]; + sortByUtf8Key(values, (entry) => entry[0]); + const deletedKeys = [...state.deletedKeys]; + sortByUtf8Key(deletedKeys, (key) => key); + const keys = collectSortedMapKeysForEncode(values, deletedKeys); + const metadataByKey = new Map(); + for (const metadata of state.metadata) { + encodeAssert(!metadataByKey.has(metadata.key), `duplicate map metadata key ${metadata.key}`); + metadataByKey.set(metadata.key, metadata); + } + encodeAssert(metadataByKey.size === keys.length, "map metadata length mismatch"); + const output = new ByteWriter(); + const postcard = new PostcardWriter(output); + writePostcardValueMap(postcard, values); + postcard.writeArray(deletedKeys, (writer, key) => writer.writeString(key)); + writePeerTable(output, state.peers); + const peerCount = BigInt(state.peers.length); + for (const key of keys) { + const metadata = metadataByKey.get(key); + encodeAssert(metadata !== undefined, `missing map metadata for key ${key}`); + assertEncodedPeerIndex(metadata.peerIndex, peerCount, "map metadata"); + writeUleb128(output, metadata.peerIndex); + writeUleb128(output, metadata.lamport); + } + return output.toUint8Array(); +} +export function decodeListStateSnapshot(bytes) { + const reader = new PostcardReader(bytes); + const values = readPostcardValues(reader); + const peers = readPeerTable(reader.input); + const columns = decodeColumnarVecMaybeWrapped(reader.input.readRemaining()); + decodeAssert(columns.length === 3, "list state must have three ID columns"); + const peerIndices = decodeDeltaRleUsize(columns[0]); + const counters = decodeDeltaRleI32(columns[1]); + const lamportSubs = decodeDeltaRleI32(columns[2]); + decodeAssert(peerIndices.length === values.length && + counters.length === values.length && + lamportSubs.length === values.length, "list state ID length mismatch"); + const peerCount = BigInt(peers.length); + const ids = peerIndices.map((peerIndex, index) => { + assertDecodedPeerIndex(peerIndex, peerCount, "list state ID"); + return { + peerIndex, + counter: counters[index], + lamportSub: lamportSubs[index], + }; + }); + return { kind: ContainerType.List, values, peers, ids }; +} +export function encodeListStateSnapshot(state) { + encodeAssert(state.ids.length === state.values.length, "list state ID length mismatch"); + const peerCount = BigInt(state.peers.length); + for (const id of state.ids) { + assertEncodedPeerIndex(id.peerIndex, peerCount, "list state ID"); + } + const ids = state.ids; + const peerIndices = new Array(ids.length); + const counters = new Array(ids.length); + const lamportSubs = new Array(ids.length); + for (let index = 0; index < ids.length; index += 1) { + const id = ids[index]; + peerIndices[index] = id.peerIndex; + counters[index] = id.counter; + lamportSubs[index] = id.lamportSub; + } + const output = new ByteWriter(); + const postcard = new PostcardWriter(output); + writePostcardValues(postcard, state.values); + writePeerTable(output, state.peers); + output.writeBytes(encodeColumnarVecWrapped([ + encodeDeltaRleUsize(peerIndices), + encodeDeltaRleI32(counters), + encodeDeltaRleI32(lamportSubs), + ])); + return output.toUint8Array(); +} +export function decodeTextStateSnapshot(bytes) { + const reader = new PostcardReader(bytes); + const text = reader.readString(); + const peers = readPeerTable(reader.input); + decodeAssert(reader.readUsize() === 3, "text state must have three encoded fields"); + const columns = readColumnarVecFrom(reader.input); + decodeAssert(columns.length === 4, "text state must have four span columns"); + const peerIndices = decodeDeltaRleUsize(columns[0]); + const counters = decodeDeltaRleI32(columns[1]); + const lamportSubs = decodeDeltaRleI32(columns[2]); + const lengths = decodeDeltaRleI32(columns[3]); + decodeAssert(peerIndices.length === lengths.length && + counters.length === lengths.length && + lamportSubs.length === lengths.length, "text state span column length mismatch"); + const peerCount = BigInt(peers.length); + const spans = lengths.map((length, index) => { + const peerIndex = peerIndices[index]; + assertDecodedPeerIndex(peerIndex, peerCount, "text state span"); + return { + peerIndex, + counter: counters[index], + lamportSub: lamportSubs[index], + length, + }; + }); + const keys = readStringVector(reader, "text mark keys"); + const markCount = reader.readUsize(); + decodeAssert(markCount <= MAX_STATE_ITEMS, "too many text marks"); + const marks = []; + for (let index = 0; index < markCount; index += 1) { + decodeAssert(reader.readUsize() === 3, "text mark must have three encoded fields"); + const keyIndex = reader.readUsize(); + const value = readPostcardValue(reader); + const info = reader.readU8(); + decodeAssert(keyIndex < keys.length, "text mark key index out of range"); + marks.push({ keyIndex, value, info }); + } + reader.assertEnd(); + validateDecodedText(text, spans, marks); + return { kind: ContainerType.Text, text, peers, spans, keys, marks }; +} +export function encodeTextStateSnapshot(state) { + validateEncodedText(state); + const spans = state.spans; + const peerIndices = new Array(spans.length); + const counters = new Array(spans.length); + const lamportSubs = new Array(spans.length); + const lengths = new Array(spans.length); + for (let index = 0; index < spans.length; index += 1) { + const span = spans[index]; + peerIndices[index] = span.peerIndex; + counters[index] = span.counter; + lamportSubs[index] = span.lamportSub; + lengths[index] = span.length; + } + const output = new ByteWriter(); + const postcard = new PostcardWriter(output); + postcard.writeString(state.text); + writePeerTable(output, state.peers); + postcard.writeUsize(3); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(peerIndices), + encodeDeltaRleI32(counters), + encodeDeltaRleI32(lamportSubs), + encodeDeltaRleI32(lengths), + ])); + postcard.writeArray(state.keys, (writer, key) => writer.writeString(key)); + postcard.writeUsize(state.marks.length); + for (const mark of state.marks) { + postcard.writeUsize(3); + postcard.writeUsize(mark.keyIndex); + writePostcardValue(postcard, mark.value); + postcard.writeU8(mark.info); + } + return output.toUint8Array(); +} +export function decodeTreeStateSnapshot(bytes) { + const input = new ByteReader(bytes); + const peers = readPeerTable(input); + const reader = new PostcardReader(input); + decodeAssert(reader.readUsize() === 4, "tree state must have four encoded fields"); + const idColumns = readColumnarVecFrom(input); + decodeAssert(idColumns.length === 2, "tree state must have two node ID columns"); + const nodePeerIndices = decodeDeltaRleUsize(idColumns[0]); + const nodeCounters = decodeDeltaRleI32(idColumns[1]); + decodeAssert(nodePeerIndices.length === nodeCounters.length, "tree state node ID column length mismatch"); + const nodeColumns = readColumnarVecFrom(input); + decodeAssert(nodeColumns.length === 5, "tree state must have five node columns"); + const parentIndices = decodeDeltaRleUsize(nodeColumns[0]); + const lastSetPeerIndices = decodeDeltaRleUsize(nodeColumns[1]); + const lastSetCounters = decodeDeltaRleI32(nodeColumns[2]); + const lastSetLamportSubs = decodeDeltaRleI32(nodeColumns[3]); + const fractionalIndexIndices = decodePostcardUsizeVector(nodeColumns[4]); + const length = nodePeerIndices.length; + decodeAssert(parentIndices.length === length && + lastSetPeerIndices.length === length && + lastSetCounters.length === length && + lastSetLamportSubs.length === length && + fractionalIndexIndices.length === length, "tree state node column length mismatch"); + const positions = decodePositionArena(reader.readBytes()); + const reserved = reader.readBytes(); + reader.assertEnd(); + const peerCount = BigInt(peers.length); + const parentIndexBound = BigInt(length + 1); + const nodes = nodePeerIndices.map((peerIndex, index) => { + const lastSetPeerIndex = lastSetPeerIndices[index]; + const parentIndexPlusTwo = parentIndices[index]; + const fractionalIndexIndex = fractionalIndexIndices[index]; + assertDecodedPeerIndex(peerIndex, peerCount, "tree node ID"); + assertDecodedPeerIndex(lastSetPeerIndex, peerCount, "tree last-set ID"); + decodeAssert(parentIndexPlusTwo <= parentIndexBound, "tree parent index out of range"); + decodeAssert(fractionalIndexIndex < positions.length, "tree fractional index out of range"); + return { + peerIndex, + counter: nodeCounters[index], + parentIndexPlusTwo, + lastSetPeerIndex, + lastSetCounter: lastSetCounters[index], + lastSetLamportSub: lastSetLamportSubs[index], + fractionalIndexIndex, + }; + }); + return { kind: ContainerType.Tree, peers, nodes, positions, reserved }; +} +export function encodeTreeStateSnapshot(state) { + validateEncodedTree(state); + const nodes = state.nodes; + const nodePeerIndices = new Array(nodes.length); + const nodeCounters = new Array(nodes.length); + const parentIndices = new Array(nodes.length); + const lastSetPeerIndices = new Array(nodes.length); + const lastSetCounters = new Array(nodes.length); + const lastSetLamportSubs = new Array(nodes.length); + const fractionalIndexIndices = new Array(nodes.length); + for (let index = 0; index < nodes.length; index += 1) { + const node = nodes[index]; + nodePeerIndices[index] = node.peerIndex; + nodeCounters[index] = node.counter; + parentIndices[index] = node.parentIndexPlusTwo; + lastSetPeerIndices[index] = node.lastSetPeerIndex; + lastSetCounters[index] = node.lastSetCounter; + lastSetLamportSubs[index] = node.lastSetLamportSub; + fractionalIndexIndices[index] = node.fractionalIndexIndex; + } + const output = new ByteWriter(); + writePeerTable(output, state.peers); + const postcard = new PostcardWriter(output); + postcard.writeUsize(4); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(nodePeerIndices), + encodeDeltaRleI32(nodeCounters), + ])); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(parentIndices), + encodeDeltaRleUsize(lastSetPeerIndices), + encodeDeltaRleI32(lastSetCounters), + encodeDeltaRleI32(lastSetLamportSubs), + encodePostcardUsizeVector(fractionalIndexIndices), + ])); + postcard.writeBytes(encodePositionArena(state.positions, { encodeEmpty: true })); + postcard.writeBytes(state.reserved); + return output.toUint8Array(); +} +export function decodeMovableListStateSnapshot(bytes) { + const reader = new PostcardReader(bytes); + const values = readPostcardValues(reader); + const peers = readPeerTable(reader.input); + decodeAssert(reader.readUsize() === 4, "movable-list state must have four encoded fields"); + const itemColumns = readColumnarVecFrom(reader.input); + decodeAssert(itemColumns.length === 3, "movable-list state item column count"); + const invisibleListItems = decodeDeltaRleUsize(itemColumns[0]); + const positionIdEqualsElementId = decodeBoolRle(itemColumns[1]); + const elementIdEqualsLastSetId = decodeBoolRle(itemColumns[2]); + decodeAssert(invisibleListItems.length === positionIdEqualsElementId.length && + invisibleListItems.length === elementIdEqualsLastSetId.length, "movable-list state item column length mismatch"); + const listIdColumns = readColumnarVecFrom(reader.input); + decodeAssert(listIdColumns.length === 3, "movable-list list ID column count"); + const listPeerIndices = decodeDeltaRleUsize(listIdColumns[0]); + const listCounters = decodeDeltaRleI32(listIdColumns[1]); + const listLamportSubs = decodeDeltaRleI32(listIdColumns[2]); + decodeAssert(listPeerIndices.length === listCounters.length && + listPeerIndices.length === listLamportSubs.length, "movable-list list ID column length mismatch"); + const elementIdColumns = readColumnarVecFrom(reader.input); + decodeAssert(elementIdColumns.length === 2, "movable-list element ID column count"); + const elementPeerIndices = decodeDeltaRleUsize(elementIdColumns[0]); + const elementLamports = decodeDeltaRleU32(elementIdColumns[1]); + decodeAssert(elementPeerIndices.length === elementLamports.length, "movable-list element ID column length mismatch"); + const lastSetIdColumns = readColumnarVecFrom(reader.input); + decodeAssert(lastSetIdColumns.length === 2, "movable-list last-set ID column count"); + const lastSetPeerIndices = decodeDeltaRleUsize(lastSetIdColumns[0]); + const lastSetLamports = decodeDeltaRleU32(lastSetIdColumns[1]); + decodeAssert(lastSetPeerIndices.length === lastSetLamports.length, "movable-list last-set ID column length mismatch"); + reader.assertEnd(); + const items = invisibleListItems.map((invisibleListItemsForItem, index) => ({ + invisibleListItems: invisibleListItemsForItem, + positionIdEqualsElementId: positionIdEqualsElementId[index], + elementIdEqualsLastSetId: elementIdEqualsLastSetId[index], + })); + const listItemIds = listPeerIndices.map((peerIndex, index) => ({ + peerIndex, + counter: listCounters[index], + lamportSub: listLamportSubs[index], + })); + const elementIds = elementPeerIndices.map((peerIndex, index) => ({ + peerIndex, + lamport: elementLamports[index], + })); + const lastSetIds = lastSetPeerIndices.map((peerIndex, index) => ({ + peerIndex, + lamport: lastSetLamports[index], + })); + validateDecodedMovableList(values.length, BigInt(peers.length), items, listItemIds, elementIds, lastSetIds); + return { + kind: ContainerType.MovableList, + values, + peers, + items, + listItemIds, + elementIds, + lastSetIds, + }; +} +export function encodeMovableListStateSnapshot(state) { + validateEncodedMovableList(state); + const items = state.items; + const invisibleListItems = new Array(items.length); + const positionIdEqualsElementId = new Array(items.length); + const elementIdEqualsLastSetId = new Array(items.length); + for (let index = 0; index < items.length; index += 1) { + const item = items[index]; + invisibleListItems[index] = item.invisibleListItems; + positionIdEqualsElementId[index] = item.positionIdEqualsElementId; + elementIdEqualsLastSetId[index] = item.elementIdEqualsLastSetId; + } + const listItemIds = state.listItemIds; + const listPeerIndices = new Array(listItemIds.length); + const listCounters = new Array(listItemIds.length); + const listLamportSubs = new Array(listItemIds.length); + for (let index = 0; index < listItemIds.length; index += 1) { + const id = listItemIds[index]; + listPeerIndices[index] = id.peerIndex; + listCounters[index] = id.counter; + listLamportSubs[index] = id.lamportSub; + } + const elementIds = state.elementIds; + const elementPeerIndices = new Array(elementIds.length); + const elementLamports = new Array(elementIds.length); + for (let index = 0; index < elementIds.length; index += 1) { + const id = elementIds[index]; + elementPeerIndices[index] = id.peerIndex; + elementLamports[index] = id.lamport; + } + const lastSetIds = state.lastSetIds; + const lastSetPeerIndices = new Array(lastSetIds.length); + const lastSetLamports = new Array(lastSetIds.length); + for (let index = 0; index < lastSetIds.length; index += 1) { + const id = lastSetIds[index]; + lastSetPeerIndices[index] = id.peerIndex; + lastSetLamports[index] = id.lamport; + } + const output = new ByteWriter(); + const postcard = new PostcardWriter(output); + writePostcardValues(postcard, state.values); + writePeerTable(output, state.peers); + postcard.writeUsize(4); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(invisibleListItems), + encodeBoolRle(positionIdEqualsElementId), + encodeBoolRle(elementIdEqualsLastSetId), + ])); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(listPeerIndices), + encodeDeltaRleI32(listCounters), + encodeDeltaRleI32(listLamportSubs), + ])); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(elementPeerIndices), + encodeDeltaRleU32(elementLamports), + ])); + output.writeBytes(encodeColumnarVec([ + encodeDeltaRleUsize(lastSetPeerIndices), + encodeDeltaRleU32(lastSetLamports), + ])); + return output.toUint8Array(); +} +export function decodeCounterStateSnapshot(bytes) { + decodeAssert(bytes.length === 8, "counter state must contain exactly eight bytes"); + return { kind: ContainerType.Counter, bits: new ByteReader(bytes).readU64LE() }; +} +export function encodeCounterStateSnapshot(state) { + const output = new ByteWriter(8); + output.writeU64LE(state.bits); + return output.toUint8Array(); +} +export function decodeContainerStateSnapshot(containerType, bytes) { + switch (containerType) { + case ContainerType.Map: + return decodeMapStateSnapshot(bytes); + case ContainerType.List: + return decodeListStateSnapshot(bytes); + case ContainerType.Text: + return decodeTextStateSnapshot(bytes); + case ContainerType.Tree: + return decodeTreeStateSnapshot(bytes); + case ContainerType.MovableList: + return decodeMovableListStateSnapshot(bytes); + case ContainerType.Counter: + return decodeCounterStateSnapshot(bytes); + default: + return { kind: containerType, payload: bytes.slice() }; + } +} +export function encodeContainerStateSnapshot(state) { + if (typeof state.kind !== "string") { + return state.payload.slice(); + } + switch (state.kind) { + case ContainerType.Map: + return encodeMapStateSnapshot(state); + case ContainerType.List: + return encodeListStateSnapshot(state); + case ContainerType.Text: + return encodeTextStateSnapshot(state); + case ContainerType.Tree: + return encodeTreeStateSnapshot(state); + case ContainerType.MovableList: + return encodeMovableListStateSnapshot(state); + case ContainerType.Counter: + return encodeCounterStateSnapshot(state); + } +} +export function decodeContainerStateWrapper(bytes) { + const input = new ByteReader(bytes); + decodeAssert(input.remaining > 0, "container state wrapper is empty"); + const containerType = containerTypeFromRawByte(input.readU8()); + const depth = readUleb128(input, U64_MAX); + const postcard = new PostcardReader(input); + const parent = readPostcardOptionalContainerId(postcard); + const state = decodeContainerStateSnapshot(containerType, input.readRemaining()); + return { containerType, depth, parent, state }; +} +export function encodeContainerStateWrapper(wrapper) { + encodeAssert(sameContainerType(wrapper.containerType, wrapper.state.kind), "container wrapper type does not match its state"); + const output = new ByteWriter(); + output.writeU8(containerTypeToRawByte(wrapper.containerType)); + writeUleb128(output, wrapper.depth); + writePostcardOptionalContainerId(new PostcardWriter(output), wrapper.parent); + output.writeBytes(encodeContainerStateSnapshot(wrapper.state)); + return output.toUint8Array(); +} +export function decodeStateSnapshotStore(bytes, options) { + if (bytes.length === 0) { + return { kind: "absent" }; + } + if (bytesEqual(bytes, EMPTY_STATE_SENTINEL)) { + return { kind: "empty" }; + } + const containers = []; + let frontiers; + for (const entry of decodeSstable(bytes, options)) { + if (bytesEqual(entry.key, FRONTIERS_KEY)) { + decodeAssert(frontiers === undefined, "duplicate state frontiers entry"); + frontiers = decodePostcardFrontiers(entry.value); + continue; + } + const id = decodeContainerId(entry.key); + const wrapper = decodeContainerStateWrapper(entry.value); + decodeAssert(sameContainerType(id.containerType, wrapper.containerType), "state container key and wrapper types differ"); + containers.push({ id, wrapper }); + } + return { kind: "sstable", frontiers, containers }; +} +export function encodeStateSnapshotStore(store, options) { + if (store.kind === "absent") { + return new Uint8Array(); + } + if (store.kind === "empty") { + return EMPTY_STATE_SENTINEL.slice(); + } + const entries = store.containers.map(({ id, wrapper }) => { + encodeAssert(sameContainerType(id.containerType, wrapper.containerType), "state container key and wrapper types differ"); + return { key: encodeContainerId(id), value: encodeContainerStateWrapper(wrapper) }; + }); + if (store.frontiers !== undefined) { + entries.push({ key: FRONTIERS_KEY, value: encodePostcardFrontiers(store.frontiers) }); + } + encodeAssert(entries.length > 0, "an SSTable state store must not be empty"); + return encodeSstable(entries, options); +} +function readPeerTable(reader) { + const length = readUlebNumber(reader, MAX_STATE_ITEMS); + decodeAssert(length <= Math.floor(reader.remaining / 8), "state peer table is truncated"); + const peers = []; + for (let index = 0; index < length; index += 1) { + peers.push(reader.readU64LE()); + } + return peers; +} +function writePeerTable(writer, peers) { + encodeAssert(peers.length <= MAX_STATE_ITEMS, "state peer table is too large"); + writeUleb128(writer, peers.length); + for (const peer of peers) { + writer.writeU64LE(peer); + } +} +function readColumnarVecFrom(reader) { + const remaining = reader.bytes.subarray(reader.position, reader.position + reader.remaining); + const [columns, rest] = takeColumnarVec(remaining); + reader.skip(remaining.length - rest.length); + return columns; +} +function readStringVector(reader, label) { + const length = reader.readUsize(); + decodeAssert(length <= MAX_STATE_ITEMS, `${label} is too large`); + decodeAssert(length <= reader.input.remaining, `${label} exceeds remaining input`); + const values = []; + for (let index = 0; index < length; index += 1) { + values.push(reader.readString()); + } + return values; +} +function decodePostcardUsizeVector(bytes) { + const reader = new PostcardReader(bytes); + const length = reader.readUsize(); + decodeAssert(length <= MAX_STATE_ITEMS, "postcard usize vector is too large"); + decodeAssert(length <= reader.input.remaining, "postcard usize vector is truncated"); + const values = []; + for (let index = 0; index < length; index += 1) { + values.push(reader.readUsize()); + } + reader.assertEnd(); + return values; +} +function encodePostcardUsizeVector(values) { + const writer = new PostcardWriter(); + writer.writeUsize(values.length); + for (const value of values) { + writer.writeUsize(value); + } + return writer.toUint8Array(); +} +function collectSortedMapKeysForDecode(values, deletedKeys) { + const keys = mergeMapKeys(values, deletedKeys); + for (let index = 1; index < keys.length; index += 1) { + decodeAssert(keys[index] !== keys[index - 1], "duplicate map state key"); + } + return keys; +} +function collectSortedMapKeysForEncode(values, deletedKeys) { + const keys = mergeMapKeys(values, deletedKeys); + for (let index = 1; index < keys.length; index += 1) { + encodeAssert(keys[index] !== keys[index - 1], "duplicate map state key"); + } + return keys; +} +function mergeMapKeys(values, deletedKeys) { + const keys = new Array(values.length + deletedKeys.length); + let offset = 0; + for (const [key] of values) { + keys[offset] = key; + offset += 1; + } + for (const key of deletedKeys) { + keys[offset] = key; + offset += 1; + } + sortByUtf8Key(keys, (key) => key); + return keys; +} +/** + * Sorts `items` in place by the UTF-8 byte order of their keys (interop + * requires UTF-8 byte order, not UTF-16 code unit order). Each key is encoded + * exactly once instead of once per comparison. + */ +function sortByUtf8Key(items, keyOf) { + const decorated = items.map((item) => ({ + bytes: textEncoder.encode(keyOf(item)), + item, + })); + decorated.sort((left, right) => compareBytes(left.bytes, right.bytes)); + for (let index = 0; index < decorated.length; index += 1) { + items[index] = decorated[index].item; + } +} +function assertDecodedPeerIndex(index, peerCount, label) { + decodeAssert(index < peerCount, `${label} peer index out of range`); +} +function assertEncodedPeerIndex(index, peerCount, label) { + encodeAssert(index >= 0n && index < peerCount, `${label} peer index out of range`); +} +function validateDecodedText(text, spans, marks) { + let markCount = 0; + let textLength = 0; + for (const span of spans) { + if (span.length === 0) { + markCount += 1; + } + else if (span.length > 0) { + textLength += span.length; + } + } + decodeAssert(markCount === marks.length, "text state mark count mismatch"); + decodeAssert(unicodeScalarLength(text) === textLength, "text state Unicode length mismatch"); +} +function validateEncodedText(state) { + let markCount = 0; + let textLength = 0; + const peerCount = BigInt(state.peers.length); + for (const span of state.spans) { + assertEncodedPeerIndex(span.peerIndex, peerCount, "text state span"); + if (span.length === 0) { + markCount += 1; + } + else if (span.length > 0) { + textLength += span.length; + } + } + encodeAssert(markCount === state.marks.length, "text state mark count mismatch"); + encodeAssert(unicodeScalarLength(state.text) === textLength, "text state Unicode length mismatch"); + for (const mark of state.marks) { + encodeAssert(mark.keyIndex >= 0 && mark.keyIndex < state.keys.length, "text mark key index out of range"); + } +} +function unicodeScalarLength(value) { + let length = 0; + for (let index = 0; index < value.length; index += 1) { + const codeUnit = value.charCodeAt(index); + if (codeUnit >= 0xd800 && + codeUnit <= 0xdbff && + index + 1 < value.length && + value.charCodeAt(index + 1) >= 0xdc00 && + value.charCodeAt(index + 1) <= 0xdfff) { + index += 1; + } + length += 1; + } + return length; +} +function validateEncodedTree(state) { + const peerCount = BigInt(state.peers.length); + const parentIndexBound = BigInt(state.nodes.length + 1); + for (const node of state.nodes) { + assertEncodedPeerIndex(node.peerIndex, peerCount, "tree node ID"); + assertEncodedPeerIndex(node.lastSetPeerIndex, peerCount, "tree last-set ID"); + encodeAssert(node.parentIndexPlusTwo >= 0n && + node.parentIndexPlusTwo <= parentIndexBound, "tree parent index out of range"); + encodeAssert(Number.isSafeInteger(node.fractionalIndexIndex) && + node.fractionalIndexIndex >= 0 && + node.fractionalIndexIndex < state.positions.length, "tree fractional index out of range"); + } +} +function validateDecodedMovableList(valueCount, peerCount, items, listItemIds, elementIds, lastSetIds) { + const visibleCount = items.length === 0 ? 0 : items.length - 1; + decodeAssert(valueCount === visibleCount, "movable-list visible value count mismatch"); + // Decoded usize values are non-negative, so the Number running total stays + // exact whenever it could equal an actual array length. + let expectedListIds = visibleCount; + for (const item of items) { + expectedListIds += Number(item.invisibleListItems); + } + decodeAssert(expectedListIds === listItemIds.length, "movable-list list ID count mismatch"); + let expectedElementIds = 0; + let expectedLastSetIds = 0; + for (let index = 1; index < items.length; index += 1) { + if (!items[index].positionIdEqualsElementId) { + expectedElementIds += 1; + } + if (!items[index].elementIdEqualsLastSetId) { + expectedLastSetIds += 1; + } + } + decodeAssert(elementIds.length === expectedElementIds, "movable-list element ID count mismatch"); + decodeAssert(lastSetIds.length === expectedLastSetIds, "movable-list last-set ID count mismatch"); + for (const id of listItemIds) { + assertDecodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); + } + for (const id of elementIds) { + assertDecodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); + } + for (const id of lastSetIds) { + assertDecodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); + } +} +function validateEncodedMovableList(state) { + const visibleCount = state.items.length === 0 ? 0 : state.items.length - 1; + encodeAssert(state.values.length === visibleCount, "movable-list visible value count mismatch"); + // Items were asserted non-negative, so the Number running total stays exact + // whenever it could equal an actual array length. + let expectedListIds = visibleCount; + for (const item of state.items) { + encodeAssert(item.invisibleListItems >= 0n, "movable-list invisible count is negative"); + expectedListIds += Number(item.invisibleListItems); + } + encodeAssert(expectedListIds === state.listItemIds.length, "movable-list list ID count mismatch"); + let expectedElementIds = 0; + let expectedLastSetIds = 0; + for (let index = 1; index < state.items.length; index += 1) { + if (!state.items[index].positionIdEqualsElementId) { + expectedElementIds += 1; + } + if (!state.items[index].elementIdEqualsLastSetId) { + expectedLastSetIds += 1; + } + } + encodeAssert(state.elementIds.length === expectedElementIds, "movable-list element ID count mismatch"); + encodeAssert(state.lastSetIds.length === expectedLastSetIds, "movable-list last-set ID count mismatch"); + const peerCount = BigInt(state.peers.length); + for (const id of state.listItemIds) { + assertEncodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); + } + for (const id of state.elementIds) { + assertEncodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); + } + for (const id of state.lastSetIds) { + assertEncodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); + } +} +function sameContainerType(left, right) { + return containerTypeToRawByte(left) === containerTypeToRawByte(right); +} diff --git a/loro-js/src/codec/types.js b/loro-js/src/codec/types.js new file mode 100644 index 000000000..fce266bb7 --- /dev/null +++ b/loro-js/src/codec/types.js @@ -0,0 +1,8 @@ +export const ContainerType = { + Map: "map", + List: "list", + Text: "text", + Tree: "tree", + MovableList: "movable-list", + Counter: "counter", +}; diff --git a/loro-js/src/codec/value.js b/loro-js/src/codec/value.js new file mode 100644 index 000000000..6ca34e7ff --- /dev/null +++ b/loro-js/src/codec/value.js @@ -0,0 +1,156 @@ +import { readPostcardContainerId, writePostcardContainerId } from "./container-id"; +import { decodeAssert } from "./errors"; +import { PostcardReader, PostcardWriter } from "./postcard"; +const MAX_VALUE_DEPTH = 1024; +export function readPostcardValue(reader, depth = 0) { + decodeAssert(depth <= MAX_VALUE_DEPTH, "postcard LoroValue is too deep", reader.input.position); + const tag = reader.readU32(); + switch (tag) { + case 0: + return { type: "null" }; + case 1: + return { type: "bool", value: reader.readBool() }; + case 2: + return { type: "double", value: reader.readF64() }; + case 3: + return { type: "i64", value: reader.readI64() }; + case 4: + return { type: "string", value: reader.readString() }; + case 5: { + const length = reader.readUsize(); + decodeAssert(length <= reader.input.remaining, "postcard LoroValue list length exceeds remaining input", reader.input.position); + const value = []; + for (let index = 0; index < length; index += 1) { + value.push(readPostcardValue(reader, depth + 1)); + } + return { type: "list", value }; + } + case 6: { + const length = reader.readUsize(); + decodeAssert(length <= Math.floor(reader.input.remaining / 2), "postcard LoroValue map length exceeds remaining input", reader.input.position); + const value = []; + for (let index = 0; index < length; index += 1) { + value.push([reader.readString(), readPostcardValue(reader, depth + 1)]); + } + return { type: "map", value }; + } + case 7: + return { type: "container", value: readPostcardContainerId(reader) }; + case 8: + return { type: "binary", value: reader.readBytes() }; + default: + decodeAssert(false, "invalid postcard LoroValue discriminant", reader.input.position); + } +} +export function writePostcardValue(writer, value, depth = 0) { + if (depth > MAX_VALUE_DEPTH) { + throw new RangeError("postcard LoroValue is too deep"); + } + switch (value.type) { + case "null": + writer.writeU32(0); + return; + case "bool": + writer.writeU32(1); + writer.writeBool(value.value); + return; + case "double": + writer.writeU32(2); + writer.writeF64(value.value); + return; + case "i64": + writer.writeU32(3); + writer.writeI64(value.value); + return; + case "string": + writer.writeU32(4); + writer.writeString(value.value); + return; + case "list": + writer.writeU32(5); + writer.writeUsize(value.value.length); + for (const item of value.value) { + writePostcardValue(writer, item, depth + 1); + } + return; + case "map": + writer.writeU32(6); + writer.writeUsize(value.value.length); + for (const [key, item] of value.value) { + writer.writeString(key); + writePostcardValue(writer, item, depth + 1); + } + return; + case "container": + writer.writeU32(7); + writePostcardContainerId(writer, value.value); + return; + case "binary": + writer.writeU32(8); + writer.writeBytes(value.value); + } +} +export function decodePostcardValue(bytes) { + const reader = new PostcardReader(bytes); + const value = readPostcardValue(reader); + reader.assertEnd(); + return value; +} +export function encodePostcardValue(value) { + const writer = new PostcardWriter(); + writePostcardValue(writer, value); + return writer.toUint8Array(); +} +export function readPostcardValues(reader) { + const length = reader.readUsize(); + decodeAssert(length <= reader.input.remaining, "postcard LoroValue vector length exceeds remaining input", reader.input.position); + const values = []; + for (let index = 0; index < length; index += 1) { + values.push(readPostcardValue(reader)); + } + return values; +} +export function writePostcardValues(writer, values) { + writer.writeUsize(values.length); + for (const value of values) { + writePostcardValue(writer, value); + } +} +export function decodePostcardValues(bytes) { + const reader = new PostcardReader(bytes); + const values = readPostcardValues(reader); + reader.assertEnd(); + return values; +} +export function encodePostcardValues(values) { + const writer = new PostcardWriter(); + writePostcardValues(writer, values); + return writer.toUint8Array(); +} +export function readPostcardValueMap(reader) { + const length = reader.readUsize(); + decodeAssert(length <= Math.floor(reader.input.remaining / 2), "postcard LoroValue map length exceeds remaining input", reader.input.position); + const entries = []; + for (let index = 0; index < length; index += 1) { + entries.push([reader.readString(), readPostcardValue(reader)]); + } + return entries; +} +export function writePostcardValueMap(writer, entries) { + writer.writeUsize(entries.length); + for (const [key, value] of entries) { + writer.writeString(key); + writePostcardValue(writer, value); + } +} +export function decodePostcardValueMap(bytes) { + const reader = new PostcardReader(bytes); + const entries = readPostcardValueMap(reader); + reader.assertEnd(); + return entries; +} +export function encodePostcardValueMap(entries) { + const writer = new PostcardWriter(); + writePostcardValueMap(writer, entries); + return writer.toUint8Array(); +} diff --git a/loro-js/src/codec/version.js b/loro-js/src/codec/version.js new file mode 100644 index 000000000..fef00a5c5 --- /dev/null +++ b/loro-js/src/codec/version.js @@ -0,0 +1,60 @@ +import { decodeAssert } from "./errors"; +import { assertId, readPostcardId, writePostcardId } from "./id"; +import { PostcardReader, PostcardWriter } from "./postcard"; +export function readPostcardVersionVector(reader) { + const length = reader.readUsize(); + decodeAssert(length <= Math.floor(reader.input.remaining / 2), "postcard version vector length exceeds remaining input", reader.input.position); + const version = []; + for (let index = 0; index < length; index += 1) { + version.push(readPostcardId(reader)); + } + return version; +} +export function writePostcardVersionVector(writer, version) { + writer.writeUsize(version.length); + for (const id of version) { + writePostcardId(writer, id); + } +} +export function decodePostcardVersionVector(bytes) { + const reader = new PostcardReader(bytes); + const version = readPostcardVersionVector(reader); + reader.assertEnd(); + return version; +} +export function encodePostcardVersionVector(version) { + const writer = new PostcardWriter(); + writePostcardVersionVector(writer, version); + return writer.toUint8Array(); +} +export function readPostcardFrontiers(reader) { + return readPostcardVersionVector(reader); +} +export function writePostcardFrontiers(writer, frontiers) { + const sorted = [...frontiers]; + for (const id of sorted) { + assertId(id); + } + sorted.sort(compareIds); + writePostcardVersionVector(writer, sorted); +} +export function decodePostcardFrontiers(bytes) { + const reader = new PostcardReader(bytes); + const frontiers = readPostcardFrontiers(reader); + reader.assertEnd(); + return frontiers; +} +export function encodePostcardFrontiers(frontiers) { + const writer = new PostcardWriter(); + writePostcardFrontiers(writer, frontiers); + return writer.toUint8Array(); +} +function compareIds(left, right) { + if (left.peer < right.peer) { + return -1; + } + if (left.peer > right.peer) { + return 1; + } + return left.counter - right.counter; +} diff --git a/loro-js/src/codec/xxhash32.js b/loro-js/src/codec/xxhash32.js new file mode 100644 index 000000000..53d3738b1 --- /dev/null +++ b/loro-js/src/codec/xxhash32.js @@ -0,0 +1,68 @@ +export const LORO_XXHASH_SEED = 1330794316; +const PRIME32_1 = 2654435761; +const PRIME32_2 = 2246822519; +const PRIME32_3 = 3266489917; +const PRIME32_4 = 668265263; +const PRIME32_5 = 374761393; +function rotateLeft(value, count) { + return ((value << count) | (value >>> (32 - count))) >>> 0; +} +function round(accumulator, input) { + let result = (accumulator + Math.imul(input, PRIME32_2)) >>> 0; + result = rotateLeft(result, 13); + return Math.imul(result, PRIME32_1) >>> 0; +} +function readU32LE(bytes, offset) { + return ((bytes[offset] | + (bytes[offset + 1] << 8) | + (bytes[offset + 2] << 16) | + (bytes[offset + 3] << 24)) >>> + 0); +} +export function xxhash32(bytes, seed = 0) { + let offset = 0; + let hash; + if (bytes.length >= 16) { + let v1 = (seed + PRIME32_1 + PRIME32_2) >>> 0; + let v2 = (seed + PRIME32_2) >>> 0; + let v3 = seed >>> 0; + let v4 = (seed - PRIME32_1) >>> 0; + const limit = bytes.length - 16; + while (offset <= limit) { + v1 = round(v1, readU32LE(bytes, offset)); + offset += 4; + v2 = round(v2, readU32LE(bytes, offset)); + offset += 4; + v3 = round(v3, readU32LE(bytes, offset)); + offset += 4; + v4 = round(v4, readU32LE(bytes, offset)); + offset += 4; + } + hash = + (rotateLeft(v1, 1) + + rotateLeft(v2, 7) + + rotateLeft(v3, 12) + + rotateLeft(v4, 18)) >>> + 0; + } + else { + hash = (seed + PRIME32_5) >>> 0; + } + hash = (hash + bytes.length) >>> 0; + while (offset + 4 <= bytes.length) { + hash = (hash + Math.imul(readU32LE(bytes, offset), PRIME32_3)) >>> 0; + hash = Math.imul(rotateLeft(hash, 17), PRIME32_4) >>> 0; + offset += 4; + } + while (offset < bytes.length) { + hash = (hash + Math.imul(bytes[offset], PRIME32_5)) >>> 0; + hash = Math.imul(rotateLeft(hash, 11), PRIME32_1) >>> 0; + offset += 1; + } + hash ^= hash >>> 15; + hash = Math.imul(hash, PRIME32_2) >>> 0; + hash ^= hash >>> 13; + hash = Math.imul(hash, PRIME32_3) >>> 0; + hash ^= hash >>> 16; + return hash >>> 0; +} diff --git a/loro-js/src/index.js b/loro-js/src/index.js new file mode 100644 index 000000000..3e3b526b1 --- /dev/null +++ b/loro-js/src/index.js @@ -0,0 +1 @@ +export * from "./runtime/index"; diff --git a/loro-js/src/runtime/containers.js b/loro-js/src/runtime/containers.js new file mode 100644 index 000000000..f676b8810 --- /dev/null +++ b/loro-js/src/runtime/containers.js @@ -0,0 +1,2597 @@ +import { bytesToHex } from "../codec/bytes"; +import { containerTypeFromHistoricalByte, containerTypeToHistoricalByte, } from "../codec/container-id"; +import { PostcardReader, PostcardWriter } from "../codec/postcard"; +import { formatContainerId, formatTreeId, parseContainerId } from "./ids"; +import { fractionalIndexBetween } from "./fractional-index"; +import { OrderedIndex } from "./ordered-index"; +import { SequenceIndex, SequenceSpan } from "./sequence-index"; +import { TextStyleIndex } from "./text-style-index"; +export class LoroContainer { + _doc; + _codecId; + _containerId; + _parentLink; + _attached; + free() { } + get id() { + return this._containerId ?? `cid:0@0:${this.kind()}`; + } + isAttached() { + return this._doc !== undefined; + } + getAttached() { + return (this.isAttached() ? this : this._attached); + } + parent() { + return this._parentLink?.container; + } + isDeleted() { + return this._doc?._isContainerDeleted(this) ?? false; + } + subscribe(listener) { + return this._doc?._subscribeContainer(this, listener) ?? (() => { }); + } + getShallowValue() { + return this.toJSON(); + } + _attach(doc, id, parent) { + this._doc = doc; + this._codecId = id; + this._containerId = formatContainerId(id); + this._parentLink = parent === undefined ? undefined : { container: parent }; + } + _setParentBinding(parent, binding) { + this._parentLink = { container: parent, binding }; + } +} +export class LoroMap extends LoroContainer { + _entries = new Map(); + _keyIndex = new OrderedIndex((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0); + _keyRecords = new Map(); + kind() { + return "Map"; + } + get(key) { + const record = this._entries.get(key); + return record === undefined || record.deleted + ? undefined + : cloneRuntimeValue(record.value); + } + set(key, value) { + if (isContainer(value)) { + throw new TypeError("use setContainer() to attach a child container"); + } + if (this._doc === undefined) { + this._applyValue(key, normalizeDetachedValue(value), { peer: 0n, lamport: 0 }); + return; + } + this._doc._mapSet(this, key, value); + } + insert(key, value) { + this.set(key, value); + } + delete(key) { + if (this._doc === undefined) { + if (this._entries.has(key)) + this._removeVisibleKey(key); + this._entries.delete(key); + return; + } + this._doc._mapDelete(this, key); + } + clear() { + for (const key of this.keys()) + this.delete(key); + } + keys() { + return this._keyIndex.values().map(({ key }) => key); + } + values() { + const records = this._keyIndex.values(); + const output = new Array(records.length); + for (let index = 0; index < records.length; index += 1) { + output[index] = this.get(records[index].key); + } + return output; + } + entries() { + const records = this._keyIndex.values(); + const output = new Array(records.length); + for (let index = 0; index < records.length; index += 1) { + output[index] = [records[index].key, this.get(records[index].key)]; + } + return output; + } + get size() { + return this._keyIndex.size; + } + setContainer(key, child) { + if (this._doc === undefined) { + this._applyValue(key, child, { peer: 0n, lamport: 0 }); + return child; + } + return this._doc._mapSetContainer(this, key, child); + } + insertContainer(key, child) { + return this.setContainer(key, child); + } + getOrCreateContainer(key, child) { + const current = this.get(key); + if (current === undefined || current === null) + return this.setContainer(key, child); + if (isContainer(current) && current.kind() === child.kind()) + return current; + throw new TypeError(`map key ${key} already contains an incompatible value`); + } + ensureMergeableMap(key) { + return this._ensureMergeable(key, "Map"); + } + ensureMergeableList(key) { + return this._ensureMergeable(key, "List"); + } + ensureMergeableMovableList(key) { + return this._ensureMergeable(key, "MovableList"); + } + ensureMergeableText(key) { + return this._ensureMergeable(key, "Text"); + } + ensureMergeableTree(key) { + return this._ensureMergeable(key, "Tree"); + } + ensureMergeableCounter(key) { + return this._ensureMergeable(key, "Counter"); + } + getLastEditor(key) { + return this._entries.get(key)?.writer.peer.toString(); + } + toJSON() { + const output = {}; + for (const { key } of this._keyIndex.values()) { + output[key] = runtimeValueToJson(this._entries.get(key).value); + } + return output; + } + getShallowValue() { + const output = {}; + for (const { key } of this._keyIndex.values()) { + output[key] = runtimeValueToShallow(this._entries.get(key).value); + } + return output; + } + _applyValue(key, value, writer, rawValue = value) { + const current = this._entries.get(key); + if (current !== undefined && compareWriters(current.writer, writer) > 0) + return; + if (current === undefined || current.deleted) + this._addVisibleKey(key); + this._entries.set(key, { value, rawValue, deleted: false, writer }); + if (value instanceof LoroContainer) { + value._setParentBinding(this, { kind: "map", key }); + } + } + _applyDelete(key, writer) { + const current = this._entries.get(key); + if (current !== undefined && compareWriters(current.writer, writer) > 0) + return; + if (current !== undefined && !current.deleted) + this._removeVisibleKey(key); + this._entries.set(key, { + value: undefined, + rawValue: undefined, + deleted: true, + writer, + }); + } + _replaceRecord(key, record) { + const current = this._entries.get(key); + if (current !== undefined && !current.deleted) + this._removeVisibleKey(key); + if (record === undefined) { + this._entries.delete(key); + return; + } + this._entries.set(key, record); + if (!record.deleted) { + this._addVisibleKey(key); + if (record.value instanceof LoroContainer) { + record.value._setParentBinding(this, { kind: "map", key }); + } + } + } + _reset() { + this._entries.clear(); + this._keyIndex.clear(); + this._keyRecords.clear(); + } + _addVisibleKey(key) { + let record = this._keyRecords.get(key); + if (record === undefined) { + record = { key }; + this._keyRecords.set(key, record); + } + this._keyIndex.add(record); + } + _removeVisibleKey(key) { + const record = this._keyRecords.get(key); + if (record !== undefined) + this._keyIndex.delete(record); + } + _ensureMergeable(key, type) { + if (this._doc === undefined) { + throw new Error("cannot ensure a mergeable child on a detached map"); + } + return this._doc._mapEnsureMergeable(this, key, type); + } +} +export class LoroList extends LoroContainer { + _sequence = new SequenceIndex(); + _detachedCounter = 0; + get _elements() { + return this._sequence.all(); + } + kind() { + return "List"; + } + get length() { + return this._sequence.visibleLength; + } + get(index) { + return cloneRuntimeValue(this._sequence.atVisible(index)?.value); + } + toArray() { + return this._visibleElements().map((element) => cloneRuntimeValue(element.value)); + } + insert(pos, value) { + this._validateInsertPosition(pos); + if (isContainer(value)) + throw new TypeError("use insertContainer() to attach a child container"); + if (this._doc === undefined) { + this._insertVisible(pos, [normalizeDetachedValue(value)], [this._nextDetachedId()], [0]); + return; + } + this._doc._listInsert(this, pos, value); + } + push(value) { + this.insert(this.length, value); + } + delete(pos, len) { + validateRange(pos, len, this.length); + if (len === 0) + return; + if (this._doc === undefined) { + for (const element of this._sequence.visibleRange(pos, pos + len)) + this._sequence.setDeleted(element, true); + return; + } + this._doc._sequenceDelete(this, pos, len); + } + clear() { + this.delete(0, this.length); + } + pop() { + if (this.length === 0) + return undefined; + const value = this.get(this.length - 1); + this.delete(this.length - 1, 1); + return value; + } + insertContainer(pos, child) { + this._validateInsertPosition(pos); + if (this._doc === undefined) { + this._insertVisible(pos, [child], [this._nextDetachedId()], [0]); + return child; + } + return this._doc._listInsertContainer(this, pos, child); + } + pushContainer(child) { + return this.insertContainer(this.length, child); + } + getIdAt(pos) { + const id = this._sequence.atVisible(pos)?.id; + return id === undefined + ? undefined + : { peer: id.peer.toString(), counter: id.counter }; + } + getCursor(pos, side = 0) { + if (!Number.isSafeInteger(pos) || pos < 0) + return undefined; + if (pos >= this.length) { + return new Cursor(this.id, this._sequence.atVisible(this.length - 1)?.id, 1, this.length); + } + return new Cursor(this.id, this._sequence.atVisible(pos).id, side, pos); + } + toJSON() { + return this._visibleElements().map((element) => runtimeValueToJson(element.value)); + } + getShallowValue() { + return this._visibleElements().map((element) => runtimeValueToShallow(element.value)); + } + _visibleElements() { + return this._sequence.visible(); + } + _visibleElementsRange(start, end) { + return this._sequence.visibleRange(start, end); + } + _valuesRange(start, end) { + return this._sequence + .visibleRange(start, end) + .map((element) => cloneRuntimeValue(element.value)); + } + _visibleElementAt(position) { + return this._sequence.atVisible(position); + } + _insertVisible(position, values, ids, lamports) { + const movable = this instanceof LoroMovableList; + const elements = values.map((value, index) => { + const id = ids[index]; + const lamport = lamports[index]; + return { + value, + id, + lamport, + deleted: false, + originLeft: undefined, + originRight: undefined, + valueHistory: movable ? [{ id, lamport, value }] : undefined, + }; + }); + this._sequence.insertAtVisible(position, elements); + this._bindChildren(elements); + } + _insertFugue(position, values, ids, lamports, causalVersion) { + const movable = this instanceof LoroMovableList; + const elements = values.map((value, index) => { + const id = ids[index]; + const elementLamport = lamports[index]; + return { + value, + id, + lamport: elementLamport, + deleted: false, + originLeft: undefined, + originRight: undefined, + valueHistory: movable ? [{ id, lamport: elementLamport, value }] : undefined, + }; + }); + insertFugueElements(this._sequence, position, elements, causalVersion, + // Moves break the origin-tree preorder used by the direct-child index. + !(this instanceof LoroMovableList)); + this._bindChildren(elements); + } + _deleteIdSpan(startId, length, deletedBy) { + if (Math.abs(length) === 1) { + const element = this._sequence.findById(startId); + if (element === undefined) + return; + this._sequence.deleteElement(element, deletedBy); + return; + } + this._sequence.deleteIdSpan(startId, length, deletedBy); + } + _validateInsertPosition(position) { + if (!Number.isSafeInteger(position) || position < 0 || position > this.length) { + throw new RangeError(`list position ${position} is out of range`); + } + } + _nextDetachedId() { + return { peer: 0n, counter: this._detachedCounter++ }; + } + _reset() { + this._sequence.reset(); + this._detachedCounter = 0; + } + _bindChildren(elements) { + for (const element of elements) { + if (element.value instanceof LoroContainer) { + element.value._setParentBinding(this, { kind: "sequence", element }); + } + } + } +} +const TEXT_NUMBER_STRIDE = 6; +const TEXT_PEER_STRIDE = 4; +const TEXT_COUNTER = 0; +const TEXT_LAMPORT = 1; +const TEXT_UTF16_END = 2; +const TEXT_ORIGIN_LEFT_COUNTER = 3; +const TEXT_ORIGIN_RIGHT_COUNTER = 4; +const TEXT_DELETED_BY_COUNTER = 5; +const TEXT_ID_PEER = 0; +const TEXT_ORIGIN_LEFT_PEER = 1; +const TEXT_ORIGIN_RIGHT_PEER = 2; +const TEXT_DELETED_BY_PEER = 3; +const NO_DELETION_IDS = []; +class TextSequenceSpan extends SequenceSpan { + #text = ""; + #length = 0; + #numbers; + #peers; + #pendingStartPeer; + #pendingStartCounter = 0; + #pendingLamport = 0; + #pendingOriginLeftPeer; + #pendingOriginLeftCounter = 0; + #pendingOriginRightPeer; + #pendingOriginRightCounter = 0; + #deletedBits = 0; + #retained; + #deletedBy; + #attributes; + #attributeMeta; + #attributeHistory; + #valueHistory; + #moveHistory; + static fromText(text, startId, lamport, originLeft, originRight) { + return TextSequenceSpan.fromTextOrigins(text, startId, lamport, originLeft?.peer, originLeft?.counter ?? 0, originRight?.peer, originRight?.counter ?? 0); + } + static fromTextOrigins(text, startId, lamport, originLeftPeer, originLeftCounter, originRightPeer, originRightCounter) { + const span = new TextSequenceSpan(); + span.#text = text; + for (const _value of text) + span.#length += 1; + span.#pendingStartPeer = startId.peer; + span.#pendingStartCounter = startId.counter; + span.#pendingLamport = lamport; + span.#pendingOriginLeftPeer = originLeftPeer; + span.#pendingOriginLeftCounter = originLeftCounter; + span.#pendingOriginRightPeer = originRightPeer; + span.#pendingOriginRightCounter = originRightCounter; + return span; + } + /** The caller must have validated ID uniqueness across the complete snapshot. */ + static fromValidatedSnapshotChunk(text, peers, counters, lamports) { + if (peers.length === 0 || + peers.length !== counters.length || + peers.length !== lamports.length) { + throw new Error("invalid text snapshot chunk columns"); + } + const span = new TextSequenceSpan(); + span.#text = text; + span.#length = peers.length; + const numbers = new Array(peers.length * TEXT_NUMBER_STRIDE).fill(0); + const peerColumns = new Array(peers.length * TEXT_PEER_STRIDE).fill(undefined); + let utf16End = 0; + let offset = 0; + for (const value of text) { + if (offset >= peers.length) { + throw new Error("text snapshot chunk contains too many Unicode scalars"); + } + utf16End += value.length; + const numberBase = offset * TEXT_NUMBER_STRIDE; + numbers[numberBase + TEXT_COUNTER] = counters[offset]; + numbers[numberBase + TEXT_LAMPORT] = lamports[offset]; + numbers[numberBase + TEXT_UTF16_END] = utf16End; + const peerBase = offset * TEXT_PEER_STRIDE; + peerColumns[peerBase + TEXT_ID_PEER] = peers[offset]; + offset += 1; + } + if (offset !== peers.length) { + throw new Error("text snapshot chunk contains too few Unicode scalars"); + } + span.#numbers = numbers; + span.#peers = peerColumns; + return span; + } + get length() { + return this.#length; + } + idsAreUnique() { + return true; + } + elementAt(offset) { + return this.#retained?.get(offset) ?? new PackedTextElement(this, offset); + } + idAt(offset) { + return { + peer: this.peerAt(offset), + counter: this.counterAt(offset), + }; + } + peerAt(offset) { + return this.#peer(offset, TEXT_ID_PEER); + } + counterAt(offset) { + return this.#number(offset, TEXT_COUNTER); + } + lamportAt(offset) { + return this.#number(offset, TEXT_LAMPORT); + } + deletedAt(offset) { + return ((this.#deletedBits >>> offset) & 1) !== 0; + } + setDeletedAt(offset, deleted) { + const bit = 1 << offset; + this.#deletedBits = deleted + ? (this.#deletedBits | bit) >>> 0 + : (this.#deletedBits & ~bit) >>> 0; + } + metricsAt(offset) { + const value = this.valueAt(offset); + return { utf16: value.length, utf8: utf8CodePointLength(value) }; + } + slice(start, end) { + this.#materializeColumns(); + const output = new TextSequenceSpan(); + output.#numbers = []; + output.#peers = []; + output.#length = end - start; + const utf16Start = start === 0 ? 0 : this.#number(start - 1, TEXT_UTF16_END); + const utf16End = end === 0 ? 0 : this.#number(end - 1, TEXT_UTF16_END); + output.#text = this.#text.slice(utf16Start, utf16End); + for (let offset = start; offset < end; offset += 1) { + const numberBase = offset * TEXT_NUMBER_STRIDE; + for (let column = 0; column < TEXT_NUMBER_STRIDE; column += 1) { + const value = this.#numbers[numberBase + column]; + output.#numbers.push(column === TEXT_UTF16_END ? value - utf16Start : value); + } + const peerBase = offset * TEXT_PEER_STRIDE; + output.#peers.push(this.#peers[peerBase], this.#peers[peerBase + 1], this.#peers[peerBase + 2], this.#peers[peerBase + 3]); + const retained = this.#retained?.get(offset); + if (retained !== undefined) { + const nextOffset = offset - start; + retained._retarget(output, nextOffset); + (output.#retained ??= new Map()).set(nextOffset, retained); + } + } + const length = end - start; + const mask = length === 32 ? 4294967295 : (1 << length) - 1; + output.#deletedBits = (this.#deletedBits >>> start) & mask; + output.#deletedBy = copyOffsetMap(this.#deletedBy, start, end); + output.#attributes = copyOffsetMap(this.#attributes, start, end); + output.#attributeMeta = copyOffsetMap(this.#attributeMeta, start, end); + output.#attributeHistory = copyOffsetMap(this.#attributeHistory, start, end); + output.#valueHistory = copyOffsetMap(this.#valueHistory, start, end); + output.#moveHistory = copyOffsetMap(this.#moveHistory, start, end); + return output; + } + append(other) { + if (!(other instanceof TextSequenceSpan)) + return false; + const oldLength = this.length; + const utf16Base = this.#text.length; + this.#materializeColumns(); + this.#text += other.#text; + for (let offset = 0; offset < other.length; offset += 1) { + for (let column = 0; column < TEXT_NUMBER_STRIDE; column += 1) { + const value = other.#number(offset, column); + this.#numbers.push(column === TEXT_UTF16_END ? value + utf16Base : value); + } + this.#peers.push(other.#peer(offset, 0), other.#peer(offset, 1), other.#peer(offset, 2), other.#peer(offset, 3)); + const retained = other.#retained?.get(offset); + if (retained !== undefined) { + const nextOffset = oldLength + offset; + retained._retarget(this, nextOffset); + (this.#retained ??= new Map()).set(nextOffset, retained); + } + } + this.#deletedBits = (this.#deletedBits | (other.#deletedBits << oldLength)) >>> 0; + this.#length += other.length; + this.#deletedBy = appendOffsetMap(this.#deletedBy, other.#deletedBy, oldLength); + this.#attributes = appendOffsetMap(this.#attributes, other.#attributes, oldLength); + this.#attributeMeta = appendOffsetMap(this.#attributeMeta, other.#attributeMeta, oldLength); + this.#attributeHistory = appendOffsetMap(this.#attributeHistory, other.#attributeHistory, oldLength); + this.#valueHistory = appendOffsetMap(this.#valueHistory, other.#valueHistory, oldLength); + this.#moveHistory = appendOffsetMap(this.#moveHistory, other.#moveHistory, oldLength); + return true; + } + deletionIdsAt(offset) { + const deletedBy = this.#deletedBy?.get(offset); + const peer = this.#peer(offset, TEXT_DELETED_BY_PEER); + if (peer === undefined) + return deletedBy ?? NO_DELETION_IDS; + const length = deletedBy?.length ?? 0; + const output = new Array(length + 1); + output[0] = { + peer, + counter: this.#number(offset, TEXT_DELETED_BY_COUNTER), + }; + for (let index = 0; index < length; index += 1) { + output[index + 1] = deletedBy[index]; + } + return output; + } + retain(offset, element) { + if (element instanceof PackedTextElement) { + (this.#retained ??= new Map()).set(offset, element); + } + } + forEachRetained(visit) { + for (const [offset, element] of this.#retained ?? []) + visit(element, offset); + } + valueAt(offset) { + const start = offset === 0 ? 0 : this.#number(offset - 1, TEXT_UTF16_END); + return this.#text.slice(start, this.#number(offset, TEXT_UTF16_END)); + } + textRange(start, end) { + const utf16Start = start === 0 ? 0 : this.#number(start - 1, TEXT_UTF16_END); + const utf16End = end === 0 ? 0 : this.#number(end - 1, TEXT_UTF16_END); + return this.#text.slice(utf16Start, utf16End); + } + setValueAt(offset, value) { + const current = this.valueAt(offset); + if (current === value) + return; + this.#materializeColumns(); + const start = offset === 0 ? 0 : this.#number(offset - 1, TEXT_UTF16_END); + const end = this.#number(offset, TEXT_UTF16_END); + this.#text = this.#text.slice(0, start) + value + this.#text.slice(end); + const difference = value.length - (end - start); + for (let index = offset; index < this.length; index += 1) { + this.#setNumber(index, TEXT_UTF16_END, this.#number(index, TEXT_UTF16_END) + difference); + } + } + originLeftAt(offset) { + return this.#idColumnAt(offset, TEXT_ORIGIN_LEFT_PEER, TEXT_ORIGIN_LEFT_COUNTER); + } + setOriginLeftAt(offset, id) { + this.#setIdColumn(offset, TEXT_ORIGIN_LEFT_PEER, TEXT_ORIGIN_LEFT_COUNTER, id); + } + originRightAt(offset) { + return this.#idColumnAt(offset, TEXT_ORIGIN_RIGHT_PEER, TEXT_ORIGIN_RIGHT_COUNTER); + } + setOriginRightAt(offset, id) { + this.#setIdColumn(offset, TEXT_ORIGIN_RIGHT_PEER, TEXT_ORIGIN_RIGHT_COUNTER, id); + } + deletedByPeerAt(offset) { + return this.#peer(offset, TEXT_DELETED_BY_PEER); + } + setDeletedByPeerAt(offset, peer) { + this.#setPeer(offset, TEXT_DELETED_BY_PEER, peer); + } + deletedByCounterAt(offset) { + return this.#peer(offset, TEXT_DELETED_BY_PEER) === undefined + ? undefined + : this.#number(offset, TEXT_DELETED_BY_COUNTER); + } + setDeletedByCounterAt(offset, counter) { + this.#setNumber(offset, TEXT_DELETED_BY_COUNTER, counter ?? 0); + } + deletedByAt(offset) { + return this.#deletedBy?.get(offset); + } + setDeletedByAt(offset, ids) { + this.#deletedBy = setOffsetMap(this.#deletedBy, offset, ids); + } + attributesAt(offset) { + return this.#attributes?.get(offset); + } + setAttributesAt(offset, value) { + this.#attributes = setOffsetMap(this.#attributes, offset, value); + } + attributeMetaAt(offset) { + return this.#attributeMeta?.get(offset); + } + setAttributeMetaAt(offset, value) { + this.#attributeMeta = setOffsetMap(this.#attributeMeta, offset, value); + } + attributeHistoryAt(offset) { + return this.#attributeHistory?.get(offset); + } + setAttributeHistoryAt(offset, value) { + this.#attributeHistory = setOffsetMap(this.#attributeHistory, offset, value); + } + valueHistoryAt(offset) { + return this.#valueHistory?.get(offset); + } + setValueHistoryAt(offset, value) { + this.#valueHistory = setOffsetMap(this.#valueHistory, offset, value); + } + moveHistoryAt(offset) { + return this.#moveHistory?.get(offset); + } + setMoveHistoryAt(offset, value) { + this.#moveHistory = setOffsetMap(this.#moveHistory, offset, value); + } + #number(offset, column) { + if (this.#numbers !== undefined) { + return this.#numbers[offset * TEXT_NUMBER_STRIDE + column]; + } + switch (column) { + case TEXT_COUNTER: + return this.#pendingStartCounter + offset; + case TEXT_LAMPORT: + return this.#pendingLamport + offset; + case TEXT_UTF16_END: { + let utf16End = 0; + let index = 0; + for (const value of this.#text) { + utf16End += value.length; + if (index === offset) + return utf16End; + index += 1; + } + throw new RangeError("text span offset is out of range"); + } + case TEXT_ORIGIN_LEFT_COUNTER: + return offset === 0 + ? this.#pendingOriginLeftCounter + : this.#pendingStartCounter + offset - 1; + case TEXT_ORIGIN_RIGHT_COUNTER: + return this.#pendingOriginRightCounter; + case TEXT_DELETED_BY_COUNTER: + return 0; + default: + throw new RangeError("text span number column is out of range"); + } + } + #setNumber(offset, column, value) { + this.#materializeColumns(); + this.#numbers[offset * TEXT_NUMBER_STRIDE + column] = value; + } + #peer(offset, column) { + if (this.#peers !== undefined) { + return this.#peers[offset * TEXT_PEER_STRIDE + column]; + } + switch (column) { + case TEXT_ID_PEER: + return this.#pendingStartPeer; + case TEXT_ORIGIN_LEFT_PEER: + return offset === 0 ? this.#pendingOriginLeftPeer : this.#pendingStartPeer; + case TEXT_ORIGIN_RIGHT_PEER: + return this.#pendingOriginRightPeer; + case TEXT_DELETED_BY_PEER: + return undefined; + default: + throw new RangeError("text span peer column is out of range"); + } + } + #setPeer(offset, column, value) { + this.#materializeColumns(); + this.#peers[offset * TEXT_PEER_STRIDE + column] = value; + } + #materializeColumns() { + if (this.#numbers !== undefined) + return; + const numbers = []; + const peers = []; + let utf16End = 0; + let offset = 0; + for (const value of this.#text) { + utf16End += value.length; + numbers.push(this.#pendingStartCounter + offset, this.#pendingLamport + offset, utf16End, offset === 0 + ? this.#pendingOriginLeftCounter + : this.#pendingStartCounter + offset - 1, this.#pendingOriginRightCounter, 0); + peers.push(this.#pendingStartPeer, offset === 0 ? this.#pendingOriginLeftPeer : this.#pendingStartPeer, this.#pendingOriginRightPeer, undefined); + offset += 1; + } + this.#numbers = numbers; + this.#peers = peers; + } + #idColumnAt(offset, peerColumn, counterColumn) { + const peer = this.#peer(offset, peerColumn); + return peer === undefined + ? undefined + : { peer, counter: this.#number(offset, counterColumn) }; + } + #setIdColumn(offset, peerColumn, counterColumn, id) { + this.#setPeer(offset, peerColumn, id?.peer); + this.#setNumber(offset, counterColumn, id?.counter ?? 0); + } +} +class PackedTextElement { + #span; + #offset; + peer; + counter; + constructor(span, offset) { + this.#span = span; + this.#offset = offset; + const id = span.idAt(offset); + this.peer = id.peer; + this.counter = id.counter; + } + get id() { + return this; + } + get lamport() { + return this.#span.lamportAt(this.#offset); + } + set lamport(_value) { + throw new TypeError("text element lamport is immutable"); + } + get value() { + return this.#span.valueAt(this.#offset); + } + set value(value) { + this.#span.setValueAt(this.#offset, value); + } + get deleted() { + return this.#span.deletedAt(this.#offset); + } + set deleted(value) { + this.#span.setDeletedAt(this.#offset, value); + } + get originLeft() { + return this.#span.originLeftAt(this.#offset); + } + set originLeft(value) { + this.#span.setOriginLeftAt(this.#offset, value); + } + get originRight() { + return this.#span.originRightAt(this.#offset); + } + set originRight(value) { + this.#span.setOriginRightAt(this.#offset, value); + } + get deletedByPeer() { + return this.#span.deletedByPeerAt(this.#offset); + } + set deletedByPeer(value) { + this.#span.setDeletedByPeerAt(this.#offset, value); + } + get deletedByCounter() { + return this.#span.deletedByCounterAt(this.#offset); + } + set deletedByCounter(value) { + this.#span.setDeletedByCounterAt(this.#offset, value); + } + get deletedBy() { + return this.#span.deletedByAt(this.#offset); + } + set deletedBy(value) { + this.#span.setDeletedByAt(this.#offset, value); + } + get attributes() { + return this.#span.attributesAt(this.#offset); + } + set attributes(value) { + this.#span.setAttributesAt(this.#offset, value); + } + get attributeMeta() { + return this.#span.attributeMetaAt(this.#offset); + } + set attributeMeta(value) { + this.#span.setAttributeMetaAt(this.#offset, value); + } + get attributeHistory() { + return this.#span.attributeHistoryAt(this.#offset); + } + set attributeHistory(value) { + this.#span.setAttributeHistoryAt(this.#offset, value); + } + get valueHistory() { + return this.#span.valueHistoryAt(this.#offset); + } + set valueHistory(value) { + this.#span.setValueHistoryAt(this.#offset, value); + } + get moveHistory() { + return this.#span.moveHistoryAt(this.#offset); + } + set moveHistory(value) { + this.#span.setMoveHistoryAt(this.#offset, value); + } + _retarget(span, offset) { + this.#span = span; + this.#offset = offset; + } +} +function setOffsetMap(map, offset, value) { + if (value === undefined) { + map?.delete(offset); + return map; + } + const output = map ?? new Map(); + output.set(offset, value); + return output; +} +function copyOffsetMap(map, start, end) { + if (map === undefined) + return undefined; + const output = new Map(); + for (const [offset, value] of map) { + if (offset >= start && offset < end) + output.set(offset - start, value); + } + return output.size === 0 ? undefined : output; +} +function appendOffsetMap(left, right, offset) { + if (right === undefined) + return left; + const output = left ?? new Map(); + for (const [index, value] of right) + output.set(offset + index, value); + return output; +} +/** Reads a visible range from sequence storage without materializing scalar views. */ +function storageTextRange(storage, start, end) { + if (storage instanceof TextSequenceSpan) + return storage.textRange(start, end); + if (storage instanceof SequenceSpan) { + throw new Error("unexpected text sequence span implementation"); + } + if (Array.isArray(storage)) { + const elements = storage; + let text = ""; + for (let offset = start; offset < end; offset += 1) + text += elements[offset].value; + return text; + } + return storage.value; +} +export class LoroText extends LoroContainer { + _sequence = new SequenceIndex((element) => ({ + utf16: element.value.length, + utf8: utf8CodePointLength(element.value), + })); + _detachedCounter = 0; + _detachedStyleCounter = 0; + _attributeHistoryComplete = true; + _styleIndex = new TextStyleIndex(); + _styleVersion; + #insertionIdContext = { + leftPeer: undefined, + leftCounter: 0, + startIndex: 0, + rightPeer: undefined, + rightCounter: 0, + }; + #validatedSnapshotSpans; + #attributeValuesCache = new WeakMap(); + get _elements() { + return this._sequence.all(); + } + kind() { + return "Text"; + } + get length() { + return this._sequence.visibleUtf16Length; + } + toString() { + return this._stringRange(0, this._sequence.visibleLength); + } + _stringRange(start, end) { + const chunks = []; + let cursor = 0; + this._sequence.forEachVisibleStorageRange((storage, rangeStart, rangeEnd) => { + const nextCursor = cursor + (rangeEnd - rangeStart); + if (nextCursor > start && end > cursor) { + chunks.push(storageTextRange(storage, start > cursor ? rangeStart + (start - cursor) : rangeStart, end < nextCursor ? rangeStart + (end - cursor) : rangeEnd)); + } + cursor = nextCursor; + return nextCursor >= end ? false : undefined; + }); + return chunks.join(""); + } + iter(callback) { + this._sequence.forEachVisibleStorageRange((storage, start, end) => callback(storageTextRange(storage, start, end)) === false ? false : undefined); + } + insert(pos, text) { + const unicodePosition = this._validateInsertPosition(pos); + if (text.length === 0) + return; + if (this._doc === undefined) { + const span = TextSequenceSpan.fromText(text, { peer: 0n, counter: this._detachedCounter }, 0, undefined, undefined); + this._detachedCounter += span.length; + this._sequence.insertSpanAtVisible(unicodePosition, span); + return; + } + this._doc._textInsert(this, unicodePosition, text); + } + push(text) { + this.insert(this.length, text); + } + insertUtf8(index, text) { + const utf16 = this.convertPos(index, "utf8", "utf16"); + if (utf16 === undefined) + throw new RangeError("UTF-8 position is out of range"); + this.insert(utf16, text); + } + delete(pos, len) { + validateRange(pos, len, this.length); + if (len === 0) + return; + const start = this._unicodePosition(pos); + const end = this._unicodePosition(pos + len); + if (this._doc === undefined) { + for (const element of this._sequence.visibleRange(start, end)) + this._sequence.setDeleted(element, true); + return; + } + this._doc._textDelete(this, start, end - start); + } + deleteUtf8(index, length) { + const start = this.convertPos(index, "utf8", "utf16"); + const end = this.convertPos(index + length, "utf8", "utf16"); + if (start === undefined || end === undefined) + throw new RangeError("UTF-8 range is out of bounds"); + this.delete(start, end - start); + } + slice(start, end) { + validateRange(start, end - start, this.length); + const unicodeStart = this._unicodePosition(start); + const unicodeEnd = this._unicodePosition(end); + return this._stringRange(unicodeStart, unicodeEnd); + } + charAt(pos) { + if (pos === this.length) + return ""; + return this._sequence.atVisible(this._unicodePosition(pos))?.value ?? ""; + } + splice(pos, len, text) { + const removed = this.slice(pos, pos + len); + this.delete(pos, len); + this.insert(pos, text); + return removed; + } + mark(range, key, value) { + validateRange(range.start, range.end - range.start, this.length); + const start = this._unicodePosition(range.start); + const end = this._unicodePosition(range.end); + if (this._doc === undefined) { + this._applyMark(start, end, key, normalizeDetachedValue(value)); + return; + } + this._doc._textMark(this, start, end, key, value); + } + unmark(range, key) { + validateRange(range.start, range.end - range.start, this.length); + const start = this._unicodePosition(range.start); + const end = this._unicodePosition(range.end); + if (!this._styleIndex.rangeHasKey(this._styleRuns(start, end), key, this._styleVersion)) { + return; + } + this.mark(range, key, null); + } + applyDelta(delta) { + let position = 0; + const marks = []; + for (const operation of delta) { + if ("insert" in operation) { + this.insert(position, operation.insert); + const length = operation.insert.length; + if (operation.attributes !== undefined) { + marks.push({ + start: position, + end: position + length, + attributes: operation.attributes, + }); + } + position += length; + } + else if ("delete" in operation) { + this.delete(position, operation.delete); + } + else { + if (operation.attributes !== undefined) { + marks.push({ + start: position, + end: position + operation.retain, + attributes: operation.attributes, + }); + } + position += operation.retain; + } + } + for (const { start, end, attributes } of marks) { + for (const [key, value] of Object.entries(attributes)) { + this.mark({ start, end }, key, value); + } + } + } + toDelta() { + return textElementsToDelta(this._visibleElements(), this.#attributeResolver()); + } + sliceDelta(start, end) { + validateRange(start, end - start, this.length); + const unicodeStart = this._unicodePosition(start); + const unicodeEnd = this._unicodePosition(end); + return textElementsToDelta(this._sequence.visibleRange(unicodeStart, unicodeEnd), this.#attributeResolver()); + } + sliceDeltaUtf8(start, end) { + const utf16Start = this.convertPos(start, "utf8", "utf16"); + const utf16End = this.convertPos(end, "utf8", "utf16"); + if (utf16Start === undefined || utf16End === undefined) { + throw new RangeError("UTF-8 range is out of bounds"); + } + return this.sliceDelta(utf16Start, utf16End); + } + convertPos(index, from, to) { + if (!isTextPosType(from) || !isTextPosType(to)) + return undefined; + const visibleLength = this._sequence.visibleLength; + const directMetric = from === "unicode" || + (from === "utf16" && this._sequence.visibleUtf16Length === visibleLength) || + (from === "utf8" && this._sequence.visibleUtf8Length === visibleLength); + const unicodeIndex = directMetric + ? Number.isSafeInteger(index) && index >= 0 && index <= visibleLength + ? index + : undefined + : this._sequence.visibleIndexAtMetricOffset(index, from); + if (unicodeIndex === undefined) + return undefined; + if (to === "unicode" || + (to === "utf16" && this._sequence.visibleUtf16Length === visibleLength) || + (to === "utf8" && this._sequence.visibleUtf8Length === visibleLength)) { + return unicodeIndex; + } + return this._sequence.metricOffsetAtVisibleIndex(unicodeIndex, to); + } + getCursor(pos, side = 0) { + if (!Number.isSafeInteger(pos) || pos < 0) + return undefined; + if (pos >= this.length) { + return new Cursor(this.id, this._sequence.atVisible(this._sequence.visibleLength - 1)?.id, 1, this.length); + } + const unicodePosition = this.convertPos(pos, "utf16", "unicode"); + if (unicodePosition === undefined) + return undefined; + return new Cursor(this.id, this._sequence.atVisible(unicodePosition).id, side, pos); + } + getEditorOf(pos) { + const unicodePosition = this.convertPos(pos, "utf16", "unicode"); + return unicodePosition === undefined + ? undefined + : this._sequence.atVisible(unicodePosition)?.id.peer.toString(); + } + update(text, _options) { + updateText(this, text); + } + updateByLine(text, options) { + this.update(text, options); + } + toJSON() { + return this.toString(); + } + getShallowValue() { + return this.toString(); + } + _visibleElements() { + return this._sequence.visible(); + } + _visibleElementsRange(start, end) { + return this._sequence.visibleRange(start, end); + } + _visibleElementAt(position) { + return this._sequence.atVisible(position); + } + _insertVisible(position, characters, ids, lamports) { + this._sequence.insertAtVisible(position, characters.map((value, index) => ({ + value, + id: ids[index], + lamport: lamports[index], + deleted: false, + originLeft: undefined, + originRight: undefined, + }))); + } + _insertValidatedSnapshotChunk(text, peers, counters, lamports) { + const span = TextSequenceSpan.fromValidatedSnapshotChunk(text, peers, counters, lamports); + if (this.#validatedSnapshotSpans !== undefined) { + this.#validatedSnapshotSpans.push(span); + } + else { + this._sequence.insertSpanAtPhysical(this._sequence.allLength, span); + } + } + _beginValidatedSnapshotLoad() { + if (this.#validatedSnapshotSpans !== undefined) { + throw new Error("text snapshot load is already active"); + } + this.#validatedSnapshotSpans = []; + } + _endValidatedSnapshotLoad() { + const spans = this.#validatedSnapshotSpans; + if (spans === undefined) + throw new Error("text snapshot load is not active"); + this.#validatedSnapshotSpans = undefined; + this._sequence.loadValidatedSpans(spans); + } + _forEachVisibleSnapshotData(appendText, appendId) { + this._sequence.forEachVisibleStorageRange((storage, start, end) => { + if (storage instanceof TextSequenceSpan) { + appendText(storage.textRange(start, end), end - start); + for (let offset = start; offset < end; offset += 1) { + appendId(storage.peerAt(offset), storage.counterAt(offset), storage.lamportAt(offset)); + } + return; + } + if (storage instanceof SequenceSpan) { + throw new Error("unexpected text sequence span implementation"); + } + const elements = Array.isArray(storage) ? storage : [storage]; + let text = ""; + for (let offset = start; offset < end; offset += 1) { + const element = elements[offset]; + text += element.value; + appendId(element.id.peer, element.id.counter, element.lamport); + } + appendText(text, end - start); + }); + } + _insertFugue(position, text, startId, lamport, causalVersion) { + const current = this._sequence.isFullyIncluded(causalVersion); + const authored = current ? undefined : this._sequence.causalView(causalVersion); + const authoredLength = current ? this._sequence.visibleLength : authored.length; + const authoredPosition = Math.min(position, authoredLength); + const needsStyleElements = !this._styleIndex.isEmpty; + const firstCodePoint = text.codePointAt(0); + const singleScalar = text.length === 1 || (text.length === 2 && firstCodePoint > 0xffff); + const useIdContext = current && !needsStyleElements && !singleScalar; + const currentContext = current && !useIdContext + ? this._sequence.visibleInsertionContext(authoredPosition) + : undefined; + const currentIdContext = useIdContext + ? this._sequence.visibleInsertionIdContext(authoredPosition, this.#insertionIdContext) + : undefined; + const authoredLeft = current + ? currentContext?.left + : authored.at(authoredPosition - 1); + const authoredRight = needsStyleElements + ? current + ? this._sequence.atVisible(authoredPosition) + : authored.at(authoredPosition) + : undefined; + const insertion = currentIdContext === undefined + ? fugueInsertion(this._sequence, position, startId, causalVersion, true, currentContext ?? { current: false, left: authoredLeft }) + : undefined; + const insertIndex = currentIdContext?.startIndex ?? insertion.insertIndex; + let inherited; + if (needsStyleElements) { + inherited = new Map(); + for (const [key, meta] of this._attributeMetasAt(authoredLeft, causalVersion)) { + if ((meta.info & 0b100) !== 0) + inherited.set(key, meta); + } + for (const [key, meta] of this._attributeMetasAt(authoredRight, causalVersion)) { + if ((meta.info & 0b010) !== 0) + inherited.set(key, meta); + } + } + let insertedLength; + if (singleScalar) { + const originLeft = currentIdContext === undefined || currentIdContext.leftPeer === undefined + ? insertion?.originLeft + : { + peer: currentIdContext.leftPeer, + counter: currentIdContext.leftCounter, + }; + const originRight = currentIdContext === undefined || currentIdContext.rightPeer === undefined + ? insertion?.originRight + : { + peer: currentIdContext.rightPeer, + counter: currentIdContext.rightCounter, + }; + const element = { + value: text, + id: { peer: startId.peer, counter: startId.counter }, + lamport, + deleted: false, + originLeft, + originRight, + }; + this._sequence.insertAtPhysical(insertIndex, [element]); + if (insertion !== undefined) { + recordFugueInsertion(this._sequence, insertion, [element.id]); + } + insertedLength = 1; + } + else { + const span = currentIdContext === undefined + ? TextSequenceSpan.fromText(text, startId, lamport, insertion.originLeft, insertion.originRight) + : TextSequenceSpan.fromTextOrigins(text, startId, lamport, currentIdContext.leftPeer, currentIdContext.leftCounter, currentIdContext.rightPeer, currentIdContext.rightCounter); + this._sequence.insertSpanAtPhysical(insertIndex, span); + if (insertion !== undefined) { + recordFugueInsertionRun(this._sequence, insertion, startId); + } + insertedLength = span.length; + } + if (inherited !== undefined && inherited.size > 0) { + const insertedRun = [ + { + start: { peer: startId.peer, counter: startId.counter }, + length: insertedLength, + }, + ]; + for (const [key, meta] of inherited) { + this._styleIndex.add(insertedRun, key, meta); + } + } + } + _deleteIdSpan(startId, length, deletedBy) { + if (Math.abs(length) === 1) { + const element = this._sequence.findById(startId); + if (element === undefined) + return; + this._sequence.deleteElement(element, deletedBy); + return; + } + this._sequence.deleteIdSpan(startId, length, deletedBy); + } + _applyMark(start, end, key, value, meta, causalVersion) { + const appliedMeta = meta ?? + ({ + startId: { peer: -1n, counter: this._detachedStyleCounter }, + lamport: this._detachedStyleCounter++, + info: 0, + value, + }); + this._styleIndex.add(this._styleRuns(start, end, causalVersion), key, appliedMeta); + if (this._styleVersion !== undefined) { + const next = new Map(this._styleVersion); + next.set(appliedMeta.startId.peer, Math.max(next.get(appliedMeta.startId.peer) ?? 0, appliedMeta.startId.counter + 1)); + this._styleVersion = next; + } + } + _styleRuns(start, end, causalVersion) { + if (causalVersion === undefined || this._sequence.isFullyIncluded(causalVersion)) { + return this._sequence.visibleIdRuns(start, end); + } + return this._sequence.causalView(causalVersion).idRuns(start, end); + } + _attributeHistoryAt(element, key) { + return this._styleIndex.historyAt(element.id, key); + } + _attributeMetasAt(element, version = this._styleVersion) { + return element === undefined + ? new Map() + : this._styleIndex.metasAt(element.id, version); + } + _attributeMetasResolver(version = this._styleVersion) { + const metasAt = this._styleIndex.resolver(version); + return (element) => metasAt(element.id); + } + _attributesAt(element) { + return this.#attributeValues(this._attributeMetasAt(element)); + } + _setStyleVersion(version) { + this._styleVersion = version === undefined ? undefined : new Map(version); + } + #attributeResolver() { + const metasAt = this._attributeMetasResolver(); + return (element) => this.#attributeValues(metasAt(element)); + } + #attributeValues(metas) { + const cached = this.#attributeValuesCache.get(metas); + if (cached !== undefined) + return cached; + const attributes = new Map(); + for (const [key, meta] of metas) { + if (meta.value !== null) + attributes.set(key, meta.value); + } + this.#attributeValuesCache.set(metas, attributes); + return attributes; + } + _validateInsertPosition(position) { + const unicodePosition = this.convertPos(position, "utf16", "unicode"); + if (unicodePosition === undefined) { + throw new RangeError(`text position ${position} is out of range`); + } + return unicodePosition; + } + _unicodePosition(position) { + const unicodePosition = this.convertPos(position, "utf16", "unicode"); + if (unicodePosition === undefined) { + throw new RangeError(`text position ${position} is not on a UTF-16 boundary`); + } + return unicodePosition; + } + _reset() { + this._sequence.reset(); + this.#validatedSnapshotSpans = undefined; + this._detachedCounter = 0; + this._detachedStyleCounter = 0; + this._attributeHistoryComplete = true; + this._styleIndex.reset(); + this._styleVersion = undefined; + } +} +export class LoroMovableList extends LoroList { + _valueHistoryComplete = true; + _moveHistoryComplete = true; + kind() { + return "MovableList"; + } + move(from, to) { + validateIndex(from, this.length); + if (!Number.isSafeInteger(to) || to < 0 || to >= this.length) { + throw new RangeError(`movable-list destination ${to} is out of range`); + } + if (from === to) + return; + if (this._doc === undefined) { + this._applyMove(from, to); + return; + } + this._doc._movableMove(this, from, to); + } + mov(from, to) { + this.move(from, to); + } + set(pos, value) { + validateIndex(pos, this.length); + if (isContainer(value)) + throw new TypeError("use setContainer() to attach a child container"); + if (this._doc === undefined) { + this._sequence.atVisible(pos).value = normalizeDetachedValue(value); + return; + } + this._doc._movableSet(this, pos, value); + } + setContainer(pos, child) { + validateIndex(pos, this.length); + if (this._doc === undefined) { + const element = this._sequence.atVisible(pos); + element.value = child; + this._bindChildren([element]); + return child; + } + return this._doc._movableSetContainer(this, pos, child); + } + getCreatorAt(pos) { + return this._sequence.atVisible(pos)?.id.peer.toString(); + } + getLastMoverAt(pos) { + return this._sequence.atVisible(pos)?.id.peer.toString(); + } + getLastEditorAt(pos) { + return this._sequence.atVisible(pos)?.id.peer.toString(); + } + _applyMove(from, to, operation, replaceExisting = false) { + const element = this._sequence.atVisible(from); + if (element === undefined) + return; + const beforePrevious = this._sequence.previousVisible(element)?.id; + const beforeNext = this._sequence.nextVisible(element)?.id; + this._sequence.moveVisible(from, to); + if (operation === undefined) + return; + const meta = { + id: operation.id, + lamport: operation.lamport, + beforePrevious, + beforeNext, + afterPrevious: this._sequence.previousVisible(element)?.id, + afterNext: this._sequence.nextVisible(element)?.id, + }; + let history = element.moveHistory; + if (history === undefined) { + history = []; + element.moveHistory = history; + } + const index = lowerBoundSequenceMoveMeta(history, meta); + const existing = history[index]; + if (existing !== undefined && + existing.id.peer === meta.id.peer && + existing.id.counter === meta.id.counter) { + if (replaceExisting) + history[index] = meta; + } + else { + history.splice(index, 0, meta); + } + } + _moveToAnchors(element, previousId, nextId) { + if (element.deleted) + return; + if (nextId === undefined) { + if (this._sequence.nextVisible(element) !== undefined) { + this._sequence.moveBefore(element, undefined); + } + return; + } + const next = this._sequence.findById(nextId); + if (next !== undefined && !next.deleted && next !== element) { + this._sequence.moveBefore(element, next); + return; + } + const previous = previousId === undefined ? undefined : this._sequence.findById(previousId); + if (previous !== undefined && !previous.deleted && previous !== element) { + const successor = this._sequence.nextVisible(previous); + if (successor !== element) + this._sequence.moveBefore(element, successor); + return; + } + if (previousId === undefined) { + const first = this._sequence.atVisible(0); + if (first !== undefined && first !== element) { + this._sequence.moveBefore(element, first); + } + } + } + _applySet(element, value, meta) { + if (meta === undefined) { + element.value = value; + this._bindChildren([element]); + return; + } + let history = element.valueHistory; + if (history === undefined) { + history = []; + element.valueHistory = history; + } + const index = lowerBoundSequenceValueMeta(history, meta); + const existing = history[index]; + if (existing === undefined || + existing.id.peer !== meta.id.peer || + existing.id.counter !== meta.id.counter) { + history.splice(index, 0, meta); + } + const winner = history.at(-1); + element.value = winner.value; + this._bindChildren([element]); + } + _reset() { + super._reset(); + this._valueHistoryComplete = true; + this._moveHistoryComplete = true; + } +} +export class LoroCounter extends LoroContainer { + _value = 0; + kind() { + return "Counter"; + } + increment(value) { + if (!Number.isFinite(value)) + throw new TypeError("counter increment must be finite"); + if (this._doc === undefined) { + this._value += value; + return; + } + this._doc._counterIncrement(this, value); + } + decrement(value) { + this.increment(-value); + } + get value() { + return this._value; + } + getValue() { + return this._value; + } + toJSON() { + return this._value; + } + getShallowValue() { + return this._value; + } + _reset() { + this._value = 0; + } +} +const DELETED_TREE_ROOT_ID = { + peer: 18446744073709551615n, + counter: 2147483647, +}; +export class LoroTree extends LoroContainer { + _nodes = new Map(); + // Parent-keyed children indexes, keyed numerically (peer -> counter) so tree + // reads/writes never stringify ids. Root children live in `_rootChildren`. + _children = new Map(); + _rootChildren; + _deletedRoots; + _fractionalIndexEnabled = true; + kind() { + return "Tree"; + } + createNode(parent, index) { + if (this._doc === undefined) + throw new Error("tree nodes can only be created on an attached tree"); + return this._doc._treeCreate(this, parent, index); + } + move(target, parent, index) { + if (this._doc === undefined) + throw new Error("tree nodes can only be moved on an attached tree"); + this._doc._treeMove(this, target, parent, index); + } + delete(target) { + if (this._doc === undefined) + return; + this._doc._treeDelete(this, target); + } + has(target) { + return this._nodes.has(target); + } + contains(target) { + return this.has(target); + } + isNodeDeleted(target) { + const node = this._nodes.get(target); + if (node === undefined) { + throw new RangeError(`tree node ${target} does not exist`); + } + return this._isRecordDeleted(node); + } + enableFractionalIndex(jitter = 0) { + if (!Number.isSafeInteger(jitter) || jitter < 0 || jitter > 0xff) { + throw new RangeError("fractional-index jitter must be an unsigned byte"); + } + this._fractionalIndexEnabled = true; + } + disableFractionalIndex() { + this._fractionalIndexEnabled = false; + } + isFractionalIndexEnabled() { + return this._fractionalIndexEnabled; + } + getNodeByID(target) { + const record = this._nodes.get(target); + return record === undefined ? undefined : new LoroTreeNode(this, record.id); + } + getNodes(options = {}) { + const records = this._nodesUnder(this._childrenOf(undefined)); + if (options.withDeleted === true) { + records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); + } + return records.map((record) => new LoroTreeNode(this, record.id)); + } + _recordsForSnapshot() { + const records = this._nodesUnder(this._childrenOf(undefined)); + records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); + return records; + } + nodes() { + return this.getNodes(); + } + roots() { + return this._childrenOf(undefined).map((record) => new LoroTreeNode(this, record.id)); + } + toArray() { + return this._childrenOf(undefined).map((record, index) => this._recordToNodeValue(record, index)); + } + toJSON() { + return this._childrenOf(undefined).map((record, index) => this._recordToValue(record, index)); + } + getShallowValue() { + return this._childrenOf(undefined).map((record, index) => this._recordToShallowValue(record, index)); + } + _childrenOf(parent) { + return this._childrenIndexFor(parent)?.values() ?? []; + } + _nodeAt(parent, index) { + const record = this._childrenIndexFor(parent)?.at(index); + return record === undefined ? undefined : new LoroTreeNode(this, record.id); + } + _rootJsonValueAt(index) { + const roots = this._childrenIndexFor(undefined); + const normalized = index < 0 ? (roots?.size ?? 0) + index : index; + const record = roots?.at(normalized); + return record === undefined + ? undefined + : this._recordToValue(record, normalized); + } + _rootCount() { + return this._childrenIndexFor(undefined)?.size ?? 0; + } + _rootJsonValuesRange(start, end) { + return (this._childrenIndexFor(undefined)?.valuesRange(start, end) ?? []).map((record, offset) => this._recordToValue(record, start + offset)); + } + _positionFor(parent, index, exclude) { + const children = this._childrenIndexFor(parent); + const excluded = exclude === undefined ? undefined : this._nodes.get(formatTreeId(exclude)); + const excludedIndex = excluded === undefined || + excluded.deleted || + !sameOptionalId(excluded.parent, parent) + ? undefined + : children?.indexOf(excluded); + const length = (children?.size ?? 0) - (excludedIndex === undefined ? 0 : 1); + const position = index ?? length; + if (!Number.isSafeInteger(position) || position < 0 || position > length) { + throw new RangeError(`tree index ${position} is out of range`); + } + // The sibling lookups below skip the slot the excluded record still + // occupies; inlined to avoid allocating a closure per call. + let before; + if (position > 0) { + const siblingIndex = excludedIndex !== undefined && position - 1 >= excludedIndex + ? position + : position - 1; + before = children.at(siblingIndex).position; + } + let after; + if (position < length) { + const siblingIndex = excludedIndex !== undefined && position >= excludedIndex + ? position + 1 + : position; + after = children.at(siblingIndex).position; + } + return fractionalIndexBetween(before, after); + } + _setRecord(record) { + this._nodes.set(formatTreeId(record.id), record); + record.data._setParentBinding(this, { kind: "tree", record }); + if (record.deleted) { + record.parent = DELETED_TREE_ROOT_ID; + this._deletedRootsIndex().add(record); + } + else { + this._childrenIndex(record.parent).add(record); + } + } + _updateRecord(record, parent, position, writer, lastMoveId) { + if (record.deleted) { + this._deletedRoots?.delete(record); + } + else { + this._childrenIndexFor(record.parent)?.delete(record); + } + record.parent = parent; + record.position = position; + record.deleted = false; + record.writer = writer; + record.lastMoveId = lastMoveId; + this._childrenIndex(parent).add(record); + } + _deleteRecord(record, writer, lastMoveId) { + if (record.deleted) { + this._deletedRoots?.delete(record); + } + else { + this._childrenIndexFor(record.parent)?.delete(record); + } + record.parent = DELETED_TREE_ROOT_ID; + record.position = Uint8Array.of(0x80); + record.deleted = true; + record.writer = writer; + record.lastMoveId = lastMoveId; + this._deletedRootsIndex().add(record); + } + _removeRecord(record) { + if (record.deleted) { + this._deletedRoots?.delete(record); + } + else { + this._childrenIndexFor(record.parent)?.delete(record); + } + record.deleted = true; + this._nodes.delete(formatTreeId(record.id)); + } + _indexOf(record) { + if (record.deleted) + return -1; + return this._childrenIndexFor(record.parent)?.indexOf(record) ?? -1; + } + _isRecordDeleted(record) { + const visited = new Set(); + let current = record; + while (current !== undefined) { + if (current.deleted) + return true; + if (current.parent === undefined) + return false; + const key = formatTreeId(current.parent); + if (visited.has(key)) + return true; + visited.add(key); + current = this._nodes.get(key); + } + return true; + } + _recordToValue(record, index = this._indexOf(record)) { + return { + id: formatTreeId(record.id), + parent: record.parent === undefined ? null : formatTreeId(record.parent), + index, + fractional_index: bytesToHex(record.position).toUpperCase(), + meta: record.data.toJSON(), + children: this._childrenOf(record.id).map((child, childIndex) => this._recordToValue(child, childIndex)), + }; + } + _recordToNodeValue(record, index = this._indexOf(record)) { + return { + id: formatTreeId(record.id), + parent: record.deleted + ? "2147483647@18446744073709551615" + : record.parent === undefined + ? undefined + : formatTreeId(record.parent), + index: record.deleted ? 0 : index, + fractionalIndex: bytesToHex(record.position).toUpperCase() || "80", + meta: record.data.toJSON(), + children: this._childrenOf(record.id).map((child, childIndex) => this._recordToNodeValue(child, childIndex)), + }; + } + _recordToShallowValue(record, index = this._indexOf(record)) { + return { + id: formatTreeId(record.id), + parent: record.deleted + ? "2147483647@18446744073709551615" + : record.parent === undefined + ? null + : formatTreeId(record.parent), + index: record.deleted ? 0 : index, + fractional_index: bytesToHex(record.position).toUpperCase(), + meta: record.data.id, + children: this._childrenOf(record.id).map((child, childIndex) => this._recordToShallowValue(child, childIndex)), + }; + } + _reset() { + this._nodes.clear(); + this._children.clear(); + this._rootChildren = undefined; + this._deletedRoots = undefined; + } + _nodesUnder(roots) { + const output = []; + const queue = roots.slice(); + for (let index = 0; index < queue.length; index += 1) { + const record = queue[index]; + output.push(record); + queue.push(...this._childrenOf(record.id)); + } + return output; + } + _childrenIndexFor(parent) { + return parent === undefined + ? this._rootChildren + : this._children.get(parent.peer)?.get(parent.counter); + } + _childrenIndex(parent) { + if (parent === undefined) { + return (this._rootChildren ??= new OrderedIndex(compareTreeRecords)); + } + let byCounter = this._children.get(parent.peer); + if (byCounter === undefined) { + byCounter = new Map(); + this._children.set(parent.peer, byCounter); + } + let index = byCounter.get(parent.counter); + if (index === undefined) { + index = new OrderedIndex(compareTreeRecords); + byCounter.set(parent.counter, index); + } + return index; + } + _deletedRootsIndex() { + return (this._deletedRoots ??= new OrderedIndex(compareTreeRecords)); + } +} +export class LoroTreeNode { + #tree; + #id; + constructor(tree, id) { + this.#tree = tree; + this.#id = id; + } + free() { } + get id() { + return formatTreeId(this.#id); + } + get data() { + return this.#record().data; + } + createNode(index) { + return this.#tree.createNode(this.id, index); + } + move(parent, index) { + this.#tree.move(this.id, parent?.id, index); + } + moveAfter(target) { + const parent = target.parent(); + const index = target.index(); + if (index === undefined) + throw new RangeError(`tree node ${target.id} is deleted`); + this.move(parent, index + 1); + } + moveBefore(target) { + const index = target.index(); + if (index === undefined) + throw new RangeError(`tree node ${target.id} is deleted`); + this.move(target.parent(), index); + } + parent() { + const parent = this.#record().parent; + return parent === undefined ? undefined : new LoroTreeNode(this.#tree, parent); + } + children() { + return this.#tree + ._childrenOf(this.#id) + .map((record) => new LoroTreeNode(this.#tree, record.id)); + } + _childAt(index) { + return this.#tree._nodeAt(this.#id, index); + } + index() { + const record = this.#record(); + const index = this.#tree._indexOf(record); + return index < 0 ? undefined : index; + } + fractionalIndex() { + const record = this.#record(); + return this.#tree._fractionalIndexEnabled && !record.deleted + ? bytesToHex(record.position).toUpperCase() + : undefined; + } + isDeleted() { + return this.#tree._isRecordDeleted(this.#record()); + } + getLastMoveId() { + const id = this.#record().lastMoveId; + return { peer: id.peer.toString(), counter: id.counter }; + } + creationId() { + return { peer: this.#id.peer.toString(), counter: this.#id.counter }; + } + creator() { + return this.#id.peer.toString(); + } + toJSON() { + return this.#tree._recordToNodeValue(this.#record()); + } + _codecId() { + return this.#id; + } + #record() { + const record = this.#tree._nodes.get(this.id); + if (record === undefined) + throw new Error(`tree node ${this.id} does not exist`); + return record; + } +} +export class Cursor { + #containerId; + #id; + #side; + #originPosition; + constructor(containerId, id, side, originPosition = 0) { + this.#containerId = containerId; + this.#id = id; + this.#side = side; + this.#originPosition = originPosition; + } + free() { } + containerId() { + return this.#containerId; + } + pos() { + return this.#id === undefined + ? undefined + : { peer: this.#id.peer.toString(), counter: this.#id.counter }; + } + side() { + return this.#side; + } + kind() { + return "Cursor"; + } + encode() { + const writer = new PostcardWriter(); + writer.writeOption(this.#id, (output, id) => { + output.writeU64(id.peer); + output.writeI32(id.counter); + }); + writePostcardCursorContainer(writer, parseContainerId(this.#containerId)); + writer.writeU32(cursorSideToVariant(this.#side)); + writer.writeUsize(this.#originPosition); + return writer.toUint8Array(); + } + static decode(bytes) { + const reader = new PostcardReader(bytes); + const id = reader.readOption((input) => ({ + peer: input.readU64(), + counter: input.readI32(), + })); + const containerId = readPostcardCursorContainer(reader); + const side = cursorSideFromVariant(reader.readU32()); + const originPosition = reader.readUsize(); + reader.assertEnd(); + return new Cursor(formatContainerId(containerId), id, side, originPosition); + } + _idValue() { + return this.#id; + } + _originPositionValue() { + return this.#originPosition; + } +} +function writePostcardCursorContainer(writer, id) { + if (id.kind === "root") { + writer.writeU32(0); + writer.writeString(id.name); + } + else { + writer.writeU32(1); + writer.writeU64(id.peer); + writer.writeI32(id.counter); + } + writer.writeU8(containerTypeToHistoricalByte(id.containerType)); +} +function readPostcardCursorContainer(reader) { + const variant = reader.readU32(); + if (variant === 0) { + const name = reader.readString(); + const containerType = containerTypeFromHistoricalByte(reader.readU8()); + return { kind: "root", name, containerType }; + } + if (variant === 1) { + const peer = reader.readU64(); + const counter = reader.readI32(); + const containerType = containerTypeFromHistoricalByte(reader.readU8()); + return { kind: "normal", peer, counter, containerType }; + } + throw new TypeError(`unsupported cursor container variant ${variant}`); +} +function cursorSideToVariant(side) { + return side === -1 ? 0 : side === 0 ? 1 : 2; +} +function cursorSideFromVariant(variant) { + if (variant === 0) + return -1; + if (variant === 1) + return 0; + if (variant === 2) + return 1; + throw new TypeError(`unsupported cursor side variant ${variant}`); +} +export function isContainer(value) { + return value instanceof LoroContainer; +} +export function getType(value) { + return isContainer(value) ? value.kind() : "Json"; +} +export function runtimeValueToJson(value) { + if (isContainer(value)) + return value.toJSON(); + if (value instanceof Uint8Array) + return value.slice(); + if (Array.isArray(value)) + return value.map(runtimeValueToJson); + if (typeof value === "object" && value !== null) { + const record = value; + const output = {}; + for (const key in record) + output[key] = runtimeValueToJson(record[key]); + return output; + } + return value; +} +export function runtimeValueToShallow(value) { + if (isContainer(value)) + return value.id; + if (value instanceof Uint8Array) + return value.slice(); + if (Array.isArray(value)) + return value.map(runtimeValueToShallow); + if (typeof value === "object" && value !== null) { + const record = value; + const output = {}; + for (const key in record) + output[key] = runtimeValueToShallow(record[key]); + return output; + } + return value; +} +export function containerDeepValueWithId(container) { + return { cid: container.id, value: containerValueWithId(container) }; +} +function containerValueWithId(container) { + if (container instanceof LoroMap) { + return Object.fromEntries(container + .keys() + .map((key) => [key, runtimeValueDeepWithId(container._entries.get(key).value)])); + } + if (container instanceof LoroText) + return container.toString(); + if (container instanceof LoroCounter) + return container.value; + if (container instanceof LoroTree) { + const visit = (record) => { + return { + id: formatTreeId(record.id), + parent: record.parent === undefined ? null : formatTreeId(record.parent), + index: container._indexOf(record), + fractional_index: bytesToHex(record.position).toUpperCase(), + meta: containerDeepValueWithId(record.data), + children: container._childrenOf(record.id).map(visit), + }; + }; + return container._childrenOf(undefined).map(visit); + } + return container + ._visibleElements() + .map((element) => runtimeValueDeepWithId(element.value)); +} +function runtimeValueDeepWithId(value) { + if (isContainer(value)) + return containerDeepValueWithId(value); + if (value instanceof Uint8Array) + return value.slice(); + if (Array.isArray(value)) + return value.map(runtimeValueDeepWithId); + if (typeof value === "object" && value !== null) { + const record = value; + const output = {}; + for (const key in record) + output[key] = runtimeValueDeepWithId(record[key]); + return output; + } + return value; +} +export function cloneRuntimeValue(value) { + if (value === undefined) + return undefined; + if (isContainer(value)) + return value; + if (value instanceof Uint8Array) + return value.slice(); + if (Array.isArray(value)) + return value.map((item) => cloneRuntimeValue(item)); + if (typeof value === "object" && value !== null) { + const record = value; + const output = {}; + for (const key in record) + output[key] = cloneRuntimeValue(record[key]); + return output; + } + return value; +} +function normalizeDetachedValue(value) { + if (value === undefined) + return null; + if (value === null || typeof value === "string" || typeof value === "boolean") + return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("Loro values must be finite numbers"); + return value; + } + if (value instanceof Uint8Array) + return value.slice(); + if (Array.isArray(value)) + return value.map(normalizeDetachedValue); + if (isContainer(value)) + return value; + if (typeof value === "object") { + if (Object.getOwnPropertySymbols(value).length > 0) { + throw new TypeError("Object keys must be strings"); + } + const output = {}; + for (const key of Object.keys(value)) { + output[key] = normalizeDetachedValue(value[key]); + } + return output; + } + throw new TypeError(`unsupported Loro value type: ${typeof value}`); +} +function insertFugueElements(sequence, position, inserted, causalVersion, useOriginIndex = true) { + if (inserted.length === 0) + return; + const insertion = fugueInsertion(sequence, position, inserted[0].id, causalVersion, useOriginIndex); + const { insertIndex, originLeft, originRight } = insertion; + for (let index = 0; index < inserted.length; index += 1) { + inserted[index].originLeft = index === 0 ? originLeft : inserted[index - 1].id; + inserted[index].originRight = originRight; + } + sequence.insertAtPhysical(insertIndex, inserted); + recordFugueInsertion(sequence, insertion, inserted.map((element) => element.id)); +} +const fugueOriginIndexes = new WeakMap(); +// Callers consume a fugueInsertion result before the next insertion, so every +// path reuses this single fixed-shape record instead of allocating per edit. +const fugueInsertionResult = { + insertIndex: 0, + originLeft: undefined, + originRight: undefined, + indexUpdate: undefined, +}; +const fugueScanVisited = new Map(); +const fugueCandidateEntries = []; +const fugueCandidateIndexes = []; +const fugueCandidateOrder = []; +const fugueImplicitEntry = { id: { peer: 0n, counter: 0 }, originLeft: undefined, originRight: undefined }; +function compareFugueCandidateOrder(left, right) { + return fugueCandidateIndexes[left] - fugueCandidateIndexes[right]; +} +function fugueInsertion(sequence, position, insertedId, causalVersion, useOriginIndex = true, positionHint) { + const current = positionHint?.current ?? sequence.isFullyIncluded(causalVersion); + const currentContext = positionHint === undefined && current + ? sequence.visibleInsertionContext(Math.min(position, sequence.visibleLength)) + : undefined; + const authoredVisible = positionHint === undefined && !current + ? sequence.causalView(causalVersion) + : undefined; + const authoredLength = current ? sequence.visibleLength : authoredVisible?.length; + const authoredPosition = Math.min(position, authoredLength ?? position); + const left = positionHint === undefined + ? current + ? currentContext?.left + : authoredVisible?.at(authoredPosition - 1) + : positionHint.left; + const originLeft = left?.id; + const startIndex = positionHint?.startIndex ?? + currentContext?.startIndex ?? + (left === undefined ? 0 : (sequence.physicalIndexOf(left) ?? -1) + 1); + const currentRight = !current + ? undefined + : positionHint?.startIndex !== undefined + ? positionHint.right + : currentContext !== undefined + ? currentContext.right + : sequence.atPhysicalRaw(startIndex); + const right = current + ? currentRight === undefined + ? undefined + : { element: currentRight, index: startIndex } + : sequence.findNextIncludedPhysical(startIndex, causalVersion); + const originRightIndex = right?.index ?? sequence.allLength; + const originRight = right?.element.id; + // There is no concurrent/future interval to order. Avoid allocating and + // maintaining an origin index for the overwhelmingly common local-edit path; + // structureVersion will force a rebuild if a later merge needs the index. + if (startIndex === originRightIndex) { + fugueInsertionResult.insertIndex = startIndex; + fugueInsertionResult.originLeft = originLeft; + fugueInsertionResult.originRight = originRight; + fugueInsertionResult.indexUpdate = undefined; + return fugueInsertionResult; + } + if (useOriginIndex) { + const indexed = indexedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId); + if (indexed !== undefined) + return indexed; + } + return scannedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId); +} +function scannedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId) { + const parentRightIndex = directRightParentIndex(sequence, originLeft, originRight); + let insertIndex = startIndex; + let scanning = false; + fugueScanVisited.clear(); + for (let index = startIndex; index < originRightIndex; index += 1) { + const other = sequence.atPhysicalRaw(index); + if (!sameOptionalId(other.originLeft, originLeft) && + (other.originLeft === undefined || + fugueScanVisited.get(other.originLeft.peer)?.has(other.originLeft.counter) !== + true)) { + break; + } + let visitedCounters = fugueScanVisited.get(other.id.peer); + if (visitedCounters === undefined) { + visitedCounters = new Set(); + fugueScanVisited.set(other.id.peer, visitedCounters); + } + visitedCounters.add(other.id.counter); + if (sameOptionalId(other.originLeft, originLeft)) { + if (sameOptionalId(other.originRight, originRight)) { + if (other.id.peer > insertedId.peer) + break; + scanning = false; + } + else { + const otherParentRightIndex = directRightParentIndex(sequence, originLeft, other.originRight); + const ordering = compareOptionalPositions(otherParentRightIndex, parentRightIndex); + if (ordering < 0) + scanning = true; + else if (ordering === 0 && other.id.peer > insertedId.peer) + break; + else + scanning = false; + } + } + if (!scanning) + insertIndex = index + 1; + } + fugueInsertionResult.insertIndex = insertIndex; + fugueInsertionResult.originLeft = originLeft; + fugueInsertionResult.originRight = originRight; + fugueInsertionResult.indexUpdate = undefined; + return fugueInsertionResult; +} +function indexedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId) { + const originIndex = getFugueOriginIndex(sequence); + // Reused candidate columns: gather the direct children inside the scanned + // interval without per-call wrapper, slice, or filtered-copy allocations. + fugueCandidateEntries.length = 0; + fugueCandidateIndexes.length = 0; + const explicit = originIndex.explicitChildren.get(optionalSequenceIdKey(originLeft)); + if (explicit !== undefined) { + for (const entry of explicit) { + const element = sequence.findByIdRaw(entry.id); + const index = element === undefined ? undefined : sequence.physicalIndexOf(element); + if (index === undefined) + return undefined; + if (index >= startIndex && index < originRightIndex) { + fugueCandidateEntries.push(entry); + fugueCandidateIndexes.push(index); + } + } + } + if (originLeft !== undefined) { + // Consecutive IDs keep their single-child edge implicit; derive it on + // lookup instead of recording an entry for every consecutive insert. + const implicit = sequence.findByIdRaw({ + peer: originLeft.peer, + counter: originLeft.counter + 1, + }); + if (implicit !== undefined && sameOptionalId(implicit.originLeft, originLeft)) { + const index = sequence.physicalIndexOf(implicit); + if (index === undefined) + return undefined; + if (index >= startIndex && index < originRightIndex) { + fugueImplicitEntry.id = implicit.id; + fugueImplicitEntry.originLeft = originLeft; + fugueImplicitEntry.originRight = implicit.originRight; + fugueCandidateEntries.push(fugueImplicitEntry); + fugueCandidateIndexes.push(index); + } + } + } + const count = fugueCandidateIndexes.length; + fugueCandidateOrder.length = count; + for (let index = 0; index < count; index += 1) + fugueCandidateOrder[index] = index; + fugueCandidateOrder.sort(compareFugueCandidateOrder); + const parentRightIndex = directRightParentIndex(sequence, originLeft, originRight); + let insertIndex = startIndex; + let scanning = false; + for (let orderIndex = 0; orderIndex < count; orderIndex += 1) { + const candidate = fugueCandidateOrder[orderIndex]; + const other = fugueCandidateEntries[candidate]; + if (sameOptionalId(other.originRight, originRight)) { + if (other.id.peer > insertedId.peer) + break; + scanning = false; + } + else { + const otherParentRightIndex = directRightParentIndex(sequence, originLeft, other.originRight); + const ordering = compareOptionalPositions(otherParentRightIndex, parentRightIndex); + if (ordering < 0) + scanning = true; + else if (ordering === 0 && other.id.peer > insertedId.peer) + break; + else + scanning = false; + } + if (!scanning) { + insertIndex = + orderIndex + 1 < count + ? fugueCandidateIndexes[fugueCandidateOrder[orderIndex + 1]] + : originRightIndex; + } + } + fugueInsertionResult.insertIndex = insertIndex; + fugueInsertionResult.originLeft = originLeft; + fugueInsertionResult.originRight = originRight; + fugueInsertionResult.indexUpdate = originIndex; + return fugueInsertionResult; +} +function getFugueOriginIndex(sequence) { + const cached = fugueOriginIndexes.get(sequence); + if (cached?.structureVersion === sequence.structureVersion) + return cached; + // Consecutive IDs form the common single-child Fugue chain, so derive those + // edges on lookup instead of allocating a map entry for every text scalar. + // Any untracked physical edit changes structureVersion and rebuilds the + // explicit branch index before it is queried again. + const rebuilt = { + structureVersion: sequence.structureVersion, + explicitChildren: new Map(), + }; + sequence.forEachPhysicalRaw((element) => { + recordFugueOriginEntry(rebuilt, element.id, element.originLeft, element.originRight); + }); + fugueOriginIndexes.set(sequence, rebuilt); + return rebuilt; +} +function recordFugueInsertion(sequence, insertion, insertedIds) { + const index = insertion.indexUpdate; + if (index === undefined) + return; + let originLeft = insertion.originLeft; + for (const id of insertedIds) { + recordFugueOriginEntry(index, id, originLeft, insertion.originRight); + originLeft = id; + } + index.structureVersion = sequence.structureVersion; +} +function recordFugueInsertionRun(sequence, insertion, startId) { + const index = insertion.indexUpdate; + if (index === undefined) + return; + recordFugueOriginEntry(index, startId, insertion.originLeft, insertion.originRight); + index.structureVersion = sequence.structureVersion; +} +function recordFugueOriginEntry(index, id, originLeft, originRight) { + if (originLeft !== undefined && + id.peer === originLeft.peer && + id.counter === originLeft.counter + 1) { + return; + } + const key = optionalSequenceIdKey(originLeft); + const children = index.explicitChildren.get(key); + const entry = { + id: { peer: id.peer, counter: id.counter }, + originLeft: originLeft === undefined + ? undefined + : { peer: originLeft.peer, counter: originLeft.counter }, + originRight: originRight === undefined + ? undefined + : { peer: originRight.peer, counter: originRight.counter }, + }; + if (children === undefined) + index.explicitChildren.set(key, [entry]); + else + children.push(entry); +} +function directRightParentIndex(sequence, originLeft, originRight) { + if (originRight === undefined) + return undefined; + const element = sequence.findByIdRaw(originRight); + if (element === undefined) + return undefined; + const index = sequence.physicalIndexOf(element); + return index !== undefined && sameOptionalId(element.originLeft, originLeft) + ? index + : undefined; +} +function compareOptionalPositions(left, right) { + if (left !== undefined && right !== undefined) + return left - right; + if (left !== undefined) + return -1; + if (right !== undefined) + return 1; + return 0; +} +function sequenceIdKey(id) { + return `${id.peer}:${id.counter}`; +} +function optionalSequenceIdKey(id) { + return id === undefined ? "root" : `id:${sequenceIdKey(id)}`; +} +function compareWriters(left, right) { + return (left.lamport - right.lamport || + (left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0)); +} +function compareSequenceValueMeta(left, right) { + return compareWriters({ peer: left.id.peer, lamport: left.lamport }, { peer: right.id.peer, lamport: right.lamport }); +} +function lowerBoundSequenceValueMeta(history, meta) { + let low = 0; + let high = history.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compareSequenceValueMeta(history[middle], meta) < 0) + low = middle + 1; + else + high = middle; + } + return low; +} +function lowerBoundSequenceMoveMeta(history, meta) { + let low = 0; + let high = history.length; + while (low < high) { + const middle = (low + high) >>> 1; + const current = history[middle]; + const order = compareWriters({ peer: current.id.peer, lamport: current.lamport }, { peer: meta.id.peer, lamport: meta.lamport }); + if (order < 0) + low = middle + 1; + else + high = middle; + } + return low; +} +function validateRange(position, length, total) { + if (!Number.isSafeInteger(position) || + !Number.isSafeInteger(length) || + position < 0 || + length < 0 || + position + length > total) { + throw new RangeError(`range ${position}..${position + length} is out of bounds for length ${total}`); + } +} +function validateIndex(index, length) { + if (!Number.isSafeInteger(index) || index < 0 || index >= length) { + throw new RangeError(`index ${index} is out of range for length ${length}`); + } +} +function sameOptionalId(left, right) { + return left === undefined || right === undefined + ? left === right + : left.peer === right.peer && left.counter === right.counter; +} +function comparePositions(left, right) { + const length = Math.min(left.length, right.length); + for (let index = 0; index < length; index += 1) { + if (left[index] !== right[index]) + return left[index] - right[index]; + } + return left.length - right.length; +} +function compareTreeRecords(left, right) { + return (comparePositions(left.position, right.position) || + compareWriters(left.writer, right.writer) || + codecIdsCompare(left.id, right.id)); +} +function codecIdsCompare(left, right) { + return left.peer < right.peer + ? -1 + : left.peer > right.peer + ? 1 + : left.counter - right.counter; +} +function textElementsToDelta(elements, attributesAt) { + const output = []; + let values = []; + let attributes; + const flush = () => { + if (values.length === 0) + return; + const insert = values.join(""); + values = []; + let jsonAttributes; + if (attributes !== undefined) { + for (const [key, value] of attributes) { + (jsonAttributes ??= {})[key] = runtimeValueToJson(value); + } + } + if (jsonAttributes === undefined) + output.push({ insert }); + else + output.push({ insert, attributes: jsonAttributes }); + }; + for (const element of elements) { + const nextAttributes = attributesAt(element); + if (values.length > 0 && !textAttributesEqual(attributes, nextAttributes)) { + flush(); + } + attributes = nextAttributes; + values.push(element.value); + } + flush(); + return output; +} +function textAttributesEqual(left, right) { + if (left === right) + return true; + if ((left?.size ?? 0) !== (right?.size ?? 0)) + return false; + for (const [key, value] of left ?? []) { + if (right?.has(key) !== true || right.get(key) !== value) + return false; + } + return true; +} +function isTextPosType(value) { + return value === "unicode" || value === "utf16" || value === "utf8"; +} +function utf8CodePointLength(value) { + const codePoint = value.codePointAt(0) ?? 0; + if (codePoint <= 0x7f) + return 1; + if (codePoint <= 0x7ff) + return 2; + if (codePoint <= 0xffff) + return 3; + return 4; +} +/** The code point whose UTF-16 end offset is `end`, ignoring pairs opened before `lowerBound`. */ +function codePointBefore(text, end, lowerBound) { + const unit = text.charCodeAt(end - 1); + if (unit >= 0xdc00 && unit <= 0xdfff && end - 2 >= lowerBound) { + const high = text.charCodeAt(end - 2); + if (high >= 0xd800 && high <= 0xdbff) { + return 65536 + ((high - 0xd800) << 10) + (unit - 0xdc00); + } + } + return unit; +} +function updateText(container, target) { + const current = container.toString(); + const currentLength = current.length; + const targetLength = target.length; + // Common prefix/suffix measured in UTF-16 offsets while comparing by code + // point, so astral characters are never split and no arrays are built. + let prefixLength = 0; + while (prefixLength < currentLength && prefixLength < targetLength) { + const codePoint = current.codePointAt(prefixLength); + if (codePoint !== target.codePointAt(prefixLength)) + break; + prefixLength += codePoint > 0xffff ? 2 : 1; + } + let suffixLength = 0; + while (suffixLength < currentLength - prefixLength && + suffixLength < targetLength - prefixLength) { + const codePoint = codePointBefore(current, currentLength - suffixLength, prefixLength); + if (codePoint !== codePointBefore(target, targetLength - suffixLength, prefixLength)) { + break; + } + suffixLength += codePoint > 0xffff ? 2 : 1; + } + container.delete(prefixLength, currentLength - prefixLength - suffixLength); + container.insert(prefixLength, target.slice(prefixLength, targetLength - suffixLength)); +} diff --git a/loro-js/src/runtime/document.js b/loro-js/src/runtime/document.js new file mode 100644 index 000000000..6281c1fe3 --- /dev/null +++ b/loro-js/src/runtime/document.js @@ -0,0 +1,6813 @@ +import packageMetadata from "../../package.json" with { type: "json" }; +import { bytesEqual, bytesToHex, hexToBytes } from "../codec/bytes"; +import { decodeChangeBlock, encodeSingleChangeBlock, validateChangeBlock, } from "../codec/change-block-codec"; +import { containerTypeFromRawByte, containerTypeToRawByte } from "../codec/container-id"; +import { decodeDocument, decodeFastSnapshotBody, decodeFastUpdatesBody, encodeDocument, encodeFastSnapshotBody, encodeFastUpdatesBody, EncodeMode, } from "../codec/document"; +import { decodeChangeBlockKey, encodeChangeBlockKey } from "../codec/id"; +import { decodeSstable, encodeSstable } from "../codec/sstable"; +import { decodeStateSnapshotStore, encodeStateSnapshotStore, } from "../codec/state-snapshot"; +import { ContainerType as CodecContainerType, } from "../codec/types"; +import { decodePostcardFrontiers, decodePostcardVersionVector, encodePostcardFrontiers, encodePostcardVersionVector, } from "../codec/version"; +import { cloneRuntimeValue, containerDeepValueWithId, Cursor, isContainer, LoroCounter, LoroList, LoroMap, LoroMovableList, LoroText, LoroTree, LoroTreeNode, runtimeValueToJson, } from "./containers"; +import { SequenceEventDiff } from "./event-diff"; +import { codecTypeToPublic, containerIdsEqual, formatContainerId, formatOpId, formatTreeId, idsEqual, isContainerId, parseContainerId, parseOpId, parsePeerId, parseTreeId, peerIdToString, publicTypeToCodec, } from "./ids"; +import { compileJsonPathEventMatcher, evaluateJsonPath } from "./jsonpath"; +import { isMergeableContainerId, mergeableMarker, newMergeableContainerId, parseMergeableMarker, } from "./mergeable"; +import { OrderedIndex } from "./ordered-index"; +import { VersionVector } from "./version-vector"; +const VERSION_KEY = Uint8Array.of(0x76, 0x76); +const FRONTIERS_KEY = Uint8Array.of(0x66, 0x72); +const START_VERSION_KEY = Uint8Array.of(0x73, 0x76); +const START_FRONTIERS_KEY = Uint8Array.of(0x73, 0x66); +let fallbackPeer = 1n; +// Scratch buffer for Counter snapshot float64 <-> bits conversion. Used +// synchronously and never retained. +const float64BitsScratch = new DataView(new ArrayBuffer(8)); +// Shared item for MovableList snapshots; the state-snapshot encoder only +// reads item fields, so every slot can reference this frozen object. +const SHARED_MOVABLE_LIST_STATE_ITEM = Object.freeze({ + invisibleListItems: 0n, + positionIdEqualsElementId: true, + elementIdEqualsLastSetId: true, +}); +// Scratch single-run list reused for synchronous, non-retaining SequenceIndex +// queries (containsIdRuns / canShowIdRunsAt). Those callees read the runs +// synchronously and never store them, so reuse is safe on this +// single-threaded runtime. Never pass these objects to anything that retains +// them. +const scratchSequenceIdRun = { start: { peer: 0n, counter: 0 }, length: 0 }; +const scratchSequenceIdRuns = [scratchSequenceIdRun]; +export class LoroDoc { + #peer = generatePeerId(); + #history = new Map(); + #historyOrder = new OrderedIndex(compareHistoryRecords); + #historyByPeer = new Map(); + #historyEndByPeer = new Map(); + #historyOperationCount = 0; + #sortedHistoryCache; + #historyFrontiers = new Map(); + #dependencyVersionCache = new WeakMap(); + #mapOperationHistory = new Map(); + #treeOperationHistory = new Map(); + #movableOrderHistory = new Map(); + #movableMovePeers = new Map(); + #containersWithOperations = new Set(); + #containerKeys = new WeakMap(); + #pendingHistory = new Map(); + #deferredSnapshotHistory; + #containers = new Map(); + #roots = new Map(); + #pending; + #nextCounter = 0; + #recordTimestamp = false; + #changeMergeInterval = 1000n; + #nextCommitOptions = {}; + #subscribers = new Set(); + #localUpdateSubscribers = new Set(); + #containerSubscribers = new Map(); + #firstCommitSubscribers = new Set(); + #preCommitSubscribers = new Set(); + #seenCommittedPeers = new Set(); + #committing = false; + #preCommitRecord; + #detached = false; + #detachedEditing = false; + #checkoutVersion; + #hideEmptyRoots = false; + #shallowStartVersion = new VersionVector(); + #shallowRootVersion = new VersionVector(); + #shallowRootFrontiers = []; + #shallowRootStore; + #textStyles = new Map([ + ["bold", "after"], + ["italic", "after"], + ["underline", "after"], + ["link", "none"], + ["highlight", "none"], + ["comment", "none"], + ["code", "none"], + ]); + #defaultTextStyle; + static fromSnapshot(snapshot) { + const doc = new LoroDoc(); + doc.import(snapshot); + return doc; + } + free() { } + get peerId() { + return this.#peer; + } + get peerIdStr() { + return peerIdToString(this.#peer); + } + setPeerId(peer) { + if (this.#pending !== undefined) + throw new Error("cannot change peer id with pending operations"); + this.#peer = parsePeerId(peer); + this.#nextCounter = this.oplogVersion().get(this.#peer) ?? 0; + } + setRecordTimestamp(enabled) { + this.#recordTimestamp = enabled; + } + setChangeMergeInterval(interval) { + this.#changeMergeInterval = numberToI64(interval); + } + setDetachedEditing(enabled) { + this.#detachedEditing = enabled; + if (enabled && this.#detached) { + this.#commit({}, true); + this.#renewPeerId(); + } + } + isDetachedEditingEnabled() { + return this.#detachedEditing; + } + configTextStyle(styles) { + this.#textStyles.clear(); + for (const [key, style] of Object.entries(styles)) { + if (key.includes(":")) + throw new TypeError("text style keys cannot contain ':'"); + assertTextStyleExpand(style.expand); + this.#textStyles.set(key, style.expand); + } + } + configDefaultTextStyle(style) { + if (style === undefined) { + this.#defaultTextStyle = undefined; + return; + } + assertTextStyleExpand(style.expand); + this.#defaultTextStyle = style.expand; + } + getMap(name) { + return this.#getContainer(name, "Map"); + } + getList(name) { + return this.#getContainer(name, "List"); + } + getMovableList(name) { + return this.#getContainer(name, "MovableList"); + } + getText(name) { + return this.#getContainer(name, "Text"); + } + getTree(name) { + return this.#getContainer(name, "Tree"); + } + getCounter(name) { + return this.#getContainer(name, "Counter"); + } + getContainerById(id) { + const parsed = parseContainerId(id); + if (parsed.kind === "root" && !isMergeableContainerId(parsed)) { + return this.#getOrCreateContainer(parsed); + } + const container = this.#containers.get(formatContainerId(parsed)); + if (container !== undefined && + isMergeableContainerId(parsed) && + this._isContainerDeleted(container) && + !this.#containerHasOperations(parsed)) { + return undefined; + } + return container; + } + hasContainer(id) { + return this.getContainerById(id) !== undefined; + } + #containerHasOperations(id) { + this.#materializeDeferredHistory(); + return this.#containersWithOperations.has(formatContainerId(id)); + } + deleteRootContainer(id) { + const parsed = parseContainerId(id); + if (parsed.kind !== "root") + throw new TypeError("only root containers can be deleted directly"); + this.#roots.delete(parsed.name); + this.#containers.delete(formatContainerId(parsed)); + } + setHideEmptyRootContainers(hide) { + this.#hideEmptyRoots = hide; + } + commit(options = {}) { + this.#commit(options, false); + } + #commit(options, preserveOptionsOnEmpty) { + if (this.#committing) + return; + const pending = this.#pending; + if (pending === undefined || pending.operations.length === 0) { + if (!preserveOptionsOnEmpty) + this.#nextCommitOptions = {}; + return; + } + const mergedOptions = { + ...this.#nextCommitOptions, + ...options, + }; + this.#nextCommitOptions = {}; + const isFirstCommit = !this.#seenCommittedPeers.has(pending.id.peer); + this.#committing = true; + try { + if (isFirstCommit) { + this.#seenCommittedPeers.add(pending.id.peer); + for (const listener of this.#firstCommitSubscribers) { + listener({ peer: peerIdToString(pending.id.peer) }); + } + } + const latestTimestamp = this.#greatestTimestampAt(pending.dependencies); + if (this.#preCommitSubscribers.size > 0) { + const initialTimestamp = maxBigInt(latestTimestamp, numberToI64(mergedOptions.timestamp ?? (this.#recordTimestamp ? Date.now() / 1000 : 0))); + const provisional = { + id: pending.id, + dependencies: pending.dependencies, + lamport: pending.lamport, + timestamp: initialTimestamp, + message: mergedOptions.message, + operations: pending.operations, + }; + const modifier = new ChangeModifier(mergedOptions); + this.#preCommitRecord = { + change: provisional, + keys: pending.keys, + keyIndices: undefined, + }; + try { + for (const listener of this.#preCommitSubscribers) { + listener({ + changeMeta: publicChange(provisional), + origin: mergedOptions.origin ?? "", + modifier, + }); + } + } + finally { + this.#preCommitRecord = undefined; + } + } + const atomLength = pending.operationLength; + const timestamp = maxBigInt(latestTimestamp, numberToI64(mergedOptions.timestamp ?? (this.#recordTimestamp ? Date.now() / 1000 : 0))); + const change = { + id: pending.id, + dependencies: pending.dependencies, + lamport: pending.lamport, + timestamp, + message: mergedOptions.message, + operations: pending.operations, + }; + this.#pending = undefined; + const committedRecord = { + change, + keys: pending.keys, + keyIndices: undefined, + }; + const previous = this.#mergeablePreviousRecord(change, this.#changeMergeInterval); + let updateRecord = committedRecord; + if (previous === undefined) { + this.#setHistoryRecord(changeKey(change.id), committedRecord, committedRecord); + } + else { + const previousLength = changeLength(previous.change); + updateRecord = appendHistoryRecord(previous, committedRecord, previousLength); + this.#appendMergedHistoryRecord(previous, updateRecord, previousLength); + } + this.#nextCounter = change.id.counter + atomLength; + if (this.#detached) { + const checkoutVersion = this.#checkoutVersion ?? new VersionVector(); + checkoutVersion.set(change.id.peer, change.id.counter + atomLength); + this.#checkoutVersion = checkoutVersion; + } + if (this.#localUpdateSubscribers.size > 0) { + const bytes = this.#encodeUpdates([updateRecord]); + for (const listener of this.#localUpdateSubscribers) + listener(bytes.slice()); + } + if (this.#hasEventSubscribers()) { + this.#emit("local", mergedOptions.origin, pending.from, this.#frontiersCodec(), pending.changedContainers, pending.beforeValues, this.#recordedEventDiffs(pending)); + } + } + catch (error) { + if (isFirstCommit) + this.#seenCommittedPeers.delete(pending.id.peer); + throw error; + } + finally { + this.#committing = false; + } + } + getPendingTxnLength() { + return this.#pending?.operationLength ?? 0; + } + setNextCommitMessage(message) { + this.#nextCommitOptions = { ...this.#nextCommitOptions, message }; + } + setNextCommitOrigin(origin) { + this.#nextCommitOptions = { ...this.#nextCommitOptions, origin }; + } + setNextCommitTimestamp(timestamp) { + this.#nextCommitOptions = { ...this.#nextCommitOptions, timestamp }; + } + setNextCommitOptions(options) { + this.#nextCommitOptions = { ...options }; + } + clearNextCommitOptions() { + this.#nextCommitOptions = {}; + } + export(mode) { + this.#commit({}, true); + if (mode.mode === "update") { + const from = mode.from ?? new VersionVector(); + const effectiveFrom = from.clone(); + effectiveFrom.merge(this.#shallowStartVersion); + const records = this.#recordsInVersionRange(effectiveFrom, this.#historyVersion()); + return this.#encodeUpdates(records); + } + if (mode.mode === "updates-in-range") { + const records = this.#recordsInSpans(mode.spans); + return this.#encodeUpdates(records); + } + if (mode.mode === "shallow-snapshot") { + return this.#encodeShallowSnapshot(mode.frontiers); + } + return this.#encodeSnapshot(); + } + exportJsonUpdates(start, end, withPeerCompression = true) { + this.#commit({}, true); + const startVersion = new VersionVector(start); + const endVersion = end === undefined || end === null ? this.#historyVersion() : new VersionVector(end); + const records = this.#recordsInVersionRange(startVersion, endVersion); + return historyRecordsToJsonSchema(records, startVersion, withPeerCompression); + } + exportJsonInIdSpan(span) { + this.#commit({}, true); + const records = this.#recordsInSpans([ + { id: { peer: span.peer, counter: span.counter }, len: span.length }, + ]); + return historyRecordsToJsonSchema(records, new VersionVector(), false, false).changes; + } + importJsonUpdates(input) { + const parsed = typeof input === "string" ? JSON.parse(input) : input; + if (parsed.schema_version !== 1) { + throw new TypeError(`unsupported JSON schema version ${String(parsed.schema_version)}`); + } + const records = jsonSchemaToHistoryRecords(parsed); + return this.import(this.#encodeUpdates(records)); + } + import(bytes) { + this.#commit({}, true); + // Event inputs are only consumed by #emit, which returns immediately when + // nobody listens, so skip building them in that case. + const emitting = this.#hasEventSubscribers(); + const before = emitting ? this.#frontiersCodec() : []; + const beforeVersion = this.#historyVersion(); + const parsed = decodeDocument(bytes); + if (this.#deferredSnapshotHistory !== undefined) { + this.#materializeDeferredHistory(); + } + const imported = []; + let integration = { added: [], pending: [] }; + let beforeValues = new Map(); + let preparedDiffs = new Map(); + let deferredChanged; + let deferredStatus; + if (parsed.mode === EncodeMode.FastUpdates) { + for (const blockBytes of decodeFastUpdatesBody(parsed.body)) { + imported.push(...this.#readChangeBlock(blockBytes)); + } + this.#assertImportsNotOutdated(imported); + integration = this.#integrateHistory(imported); + const { added } = integration; + if (added.length > 0 && !this.#detached) { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + if (recording !== undefined) { + this.#recordConcurrentTreeSubjects(recording, added, beforeVersion); + } + this.#applyRecords(added, recording); + this.#canonicalizeImportedMovableMoves(added, recording); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + } + else { + const snapshot = decodeFastSnapshotBody(parsed.body); + const oplogEntries = decodeSstable(snapshot.oplog.slice()); + let encodedStartVersion; + let encodedStartFrontiers; + let encodedEndVersion; + let encodedEndFrontiers; + for (const entry of oplogEntries) { + if (bytesEqual(entry.key, START_VERSION_KEY)) { + encodedStartVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); + } + else if (bytesEqual(entry.key, START_FRONTIERS_KEY)) { + encodedStartFrontiers = decodePostcardFrontiers(entry.value); + } + else if (bytesEqual(entry.key, VERSION_KEY)) { + encodedEndVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); + } + else if (bytesEqual(entry.key, FRONTIERS_KEY)) { + encodedEndFrontiers = decodePostcardFrontiers(entry.value); + } + } + const initializeFromSnapshot = this.#history.size === 0 && + this.#pendingHistory.size === 0 && + this.#shallowRootStore === undefined; + const isShallowSnapshot = snapshot.shallowRootState.length > 0; + let rootStore; + let stagedShallowStartVersion; + let stagedShallowRootVersion; + let stagedShallowRootFrontiers; + let canonicalStartMetadata = !isShallowSnapshot && + encodedStartVersion === undefined && + encodedStartFrontiers === undefined; + if (initializeFromSnapshot && isShallowSnapshot) { + rootStore = decodeStateSnapshotStore(snapshot.shallowRootState.slice()); + if (rootStore.kind !== "sstable") { + throw new Error("shallow snapshot root state must be an SSTable"); + } + if (rootStore.frontiers !== undefined && + encodedStartFrontiers !== undefined && + !frontierSetsEqual(rootStore.frontiers.map(formatOpId), encodedStartFrontiers.map(formatOpId))) { + throw new Error("shallow snapshot start frontiers do not match root state"); + } + const startFrontiers = encodedStartFrontiers ?? rootStore.frontiers ?? []; + const startVersion = encodedStartVersion ?? new VersionVector(); + const rootVersion = startVersion.clone(); + for (const frontier of startFrontiers) { + rootVersion.set(frontier.peer, frontier.counter + 1); + } + stagedShallowStartVersion = startVersion; + stagedShallowRootVersion = rootVersion; + stagedShallowRootFrontiers = startFrontiers.map((id) => ({ ...id })); + canonicalStartMetadata = + encodedStartVersion !== undefined && encodedStartFrontiers !== undefined; + } + const stateStore = decodeStateSnapshotStore(snapshot.state); + const hydratedStore = rootStore === undefined + ? stateStore + : mergeStateSnapshotStores(rootStore, stateStore); + const preparedTextSnapshotIds = validateStateSnapshotTextIds(hydratedStore); + if (stagedShallowRootVersion !== undefined && + encodedEndVersion !== undefined && + !versionIncludes(encodedEndVersion, stagedShallowRootVersion)) { + throw new Error("shallow snapshot root version exceeds its end version"); + } + const installStagedShallowRoot = () => { + if (rootStore === undefined || + stagedShallowStartVersion === undefined || + stagedShallowRootVersion === undefined || + stagedShallowRootFrontiers === undefined) { + return; + } + this.#shallowStartVersion = stagedShallowStartVersion; + this.#shallowRootVersion = stagedShallowRootVersion; + this.#shallowRootFrontiers = stagedShallowRootFrontiers; + this.#shallowRootStore = rootStore; + }; + const canDeferHistory = initializeFromSnapshot && + !this.#detached && + canonicalStartMetadata && + encodedEndVersion !== undefined && + encodedEndFrontiers !== undefined && + stateStore.kind === "sstable" && + hydratedStore.kind === "sstable"; + if (canDeferHistory) { + const endVersion = encodedEndVersion; + const endFrontiers = encodedEndFrontiers; + this.#validateDeferredFrontierBlocks(oplogEntries, endVersion, endFrontiers); + installStagedShallowRoot(); + if (emitting) { + deferredChanged = new Set(hydratedStore.containers.map(({ id }) => formatContainerId(id))); + beforeValues = this.#captureContainerEventValues(deferredChanged); + } + this.#hydrateState(hydratedStore, preparedTextSnapshotIds); + this.#deferredSnapshotHistory = { + entries: oplogEntries, + endVersion, + frontiers: endFrontiers.map((id) => ({ ...id })), + operationCount: versionDistance(stagedShallowStartVersion ?? this.#shallowStartVersion, endVersion), + }; + for (const { peer } of endVersion._codecEntriesUnsorted()) { + this.#seenCommittedPeers.add(peer); + } + deferredStatus = importStatusBetweenVersions(beforeVersion, stagedShallowStartVersion ?? this.#shallowStartVersion, endVersion); + } + else { + for (const entry of oplogEntries) { + if (entry.key.length !== 12) + continue; + const expected = decodeChangeBlockKey(entry.key); + const records = this.#readChangeBlock(entry.value); + if (records[0] !== undefined && !idsEqual(records[0].change.id, expected)) { + throw new Error("snapshot change key does not match its block"); + } + imported.push(...records); + } + installStagedShallowRoot(); + if (!initializeFromSnapshot) + this.#assertImportsNotOutdated(imported); + integration = this.#integrateHistory(imported); + if (!this.#detached && initializeFromSnapshot) { + if (this.#hasEventSubscribers()) { + beforeValues = this.#captureEventValues(integration.added); + } + if (rootStore !== undefined && stateStore.kind === "empty") { + this.#rebuildFromHistory(); + } + else { + this.#hydrateState(hydratedStore, preparedTextSnapshotIds); + } + } + else if (!this.#detached && integration.added.length > 0) { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + if (recording !== undefined) { + this.#recordConcurrentTreeSubjects(recording, integration.added, beforeVersion); + } + this.#applyRecords(integration.added, recording); + this.#canonicalizeImportedMovableMoves(integration.added, recording); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + } + } + this.#nextCounter = this.oplogVersion().get(this.#peer) ?? 0; + const changed = this.#detached || !emitting + ? new Set() + : (deferredChanged ?? + new Set(integration.added.flatMap(({ change }) => change.operations.map((op) => formatContainerId(op.container))))); + this.#emit("import", undefined, before, emitting ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); + return deferredStatus ?? importStatus(integration.added, integration.pending); + } + importBatch(blobs) { + if (blobs.length === 0) + return { success: new Map(), pending: null }; + if (blobs.length === 1) + return this.import(blobs[0]); + this.#commit({}, true); + this.#materializeDeferredHistory(); + const emitting = this.#hasEventSubscribers(); + const before = emitting ? this.#frontiersCodec() : []; + const ordered = blobs + .map((blob) => this.#decodeImportData(decodeDocument(blob))) + .sort((left, right) => Number(left.mode === EncodeMode.FastUpdates) - + Number(right.mode === EncodeMode.FastUpdates) || + right.records.length - left.records.length); + const initializeFromSnapshot = this.#history.size === 0 && + this.#pendingHistory.size === 0 && + this.#shallowRootStore === undefined; + const snapshotSeed = initializeFromSnapshot + ? ordered.find(({ snapshot }) => snapshot !== undefined) + : undefined; + let rootStore; + let stagedShallowRoot; + if (snapshotSeed?.snapshot !== undefined && + snapshotSeed.snapshot.shallowRootState.length > 0) { + rootStore = decodeStateSnapshotStore(snapshotSeed.snapshot.shallowRootState); + if (rootStore.kind !== "sstable") { + throw new Error("shallow snapshot root state must be an SSTable"); + } + const startFrontiers = snapshotSeed.startFrontiers ?? rootStore.frontiers ?? []; + const startVersion = snapshotSeed.startVersion ?? new VersionVector(); + const rootVersion = startVersion.clone(); + for (const frontier of startFrontiers) { + rootVersion.set(frontier.peer, frontier.counter + 1); + } + stagedShallowRoot = { + startVersion, + rootVersion, + frontiers: startFrontiers.map((id) => ({ ...id })), + }; + } + let snapshotStateStore; + let snapshotHydratedStore; + let preparedTextSnapshotIds; + if (snapshotSeed?.snapshot !== undefined) { + snapshotStateStore = decodeStateSnapshotStore(snapshotSeed.snapshot.state); + snapshotHydratedStore = + rootStore === undefined + ? snapshotStateStore + : mergeStateSnapshotStores(rootStore, snapshotStateStore); + preparedTextSnapshotIds = validateStateSnapshotTextIds(snapshotHydratedStore); + } + if (rootStore !== undefined && stagedShallowRoot !== undefined) { + this.#shallowStartVersion = stagedShallowRoot.startVersion; + this.#shallowRootVersion = stagedShallowRoot.rootVersion; + this.#shallowRootFrontiers = stagedShallowRoot.frontiers; + this.#shallowRootStore = rootStore; + } + for (const decoded of ordered) { + if (decoded !== snapshotSeed) + this.#assertImportsNotOutdated(decoded.records); + } + const integration = this.#integrateHistory(ordered.flatMap(({ records }) => records)); + let beforeValues = new Map(); + let preparedDiffs = new Map(); + if (integration.added.length > 0 && !this.#detached) { + if (snapshotSeed?.snapshot !== undefined) { + if (this.#hasEventSubscribers()) { + beforeValues = this.#captureEventValues(integration.added); + } + if (rootStore !== undefined && snapshotStateStore.kind === "empty") { + this.#rebuildFromHistory(); + } + else { + this.#hydrateState(snapshotHydratedStore, preparedTextSnapshotIds); + const snapshotVersion = snapshotSeed.endVersion?.clone() ?? + historyVersionForRecords(snapshotSeed.records, snapshotSeed.startVersion); + const forwardRecords = this.#recordsInVersionRange(snapshotVersion, this.#historyVersion()); + this.#applyRecords(forwardRecords); + this.#canonicalizeImportedMovableMoves(forwardRecords); + } + } + else { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + this.#applyRecords(integration.added, recording); + this.#canonicalizeImportedMovableMoves(integration.added, recording); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + } + this.#nextCounter = this.oplogVersion().get(this.#peer) ?? 0; + const changed = this.#detached || !emitting + ? new Set() + : changedContainerIds(integration.added); + this.#emit("import", undefined, before, emitting ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); + return importStatus(integration.added, integration.pending); + } + importUpdateBatch(blobs) { + return this.importBatch(blobs); + } + toJSON() { + const output = {}; + for (const [name, container] of this.#roots) { + const value = container.toJSON(); + if (this.#hideEmptyRoots && isEmptyJson(value)) + continue; + output[name] = value; + } + return output; + } + toJsonWithReplacer(replacer) { + const processed = new Set(); + const run = (value) => { + if (Array.isArray(value)) { + return value.flatMap((item, index) => { + const next = visit(index, item); + return next === undefined ? [] : [next]; + }); + } + if (typeof value === "object" && value !== null && !(value instanceof Uint8Array)) { + return Object.fromEntries(Object.entries(value).flatMap(([key, item]) => { + const next = visit(key, item); + return next === undefined ? [] : [[key, next]]; + })); + } + return value; + }; + const visit = (key, value) => { + if (typeof value === "string" && + isContainerId(value) && + !processed.has(value)) { + const id = value; + processed.add(id); + const container = this.getContainerById(id); + if (container === undefined) + throw new RangeError(`container ${id} does not exist`); + const replaced = replacer(key, container); + if (replaced === container) + return run(container.getShallowValue()); + if (isContainer(replaced)) { + throw new TypeError("replacer cannot substitute a different container"); + } + return replaced === undefined ? undefined : run(replaced); + } + if (typeof value === "object" && value !== null && !(value instanceof Uint8Array)) { + return run(value); + } + const replaced = replacer(key, value); + if (isContainer(replaced)) { + throw new TypeError("replacer cannot introduce a container"); + } + return replaced; + }; + return run(this.getShallowValue()); + } + diff(from, to, forJson = true) { + this.#commit({}, true); + const fromVersion = this.#versionForExistingFrontiers(from); + const toVersion = this.#versionForExistingFrontiers(to); + this.#assertVersionNotBeforeShallowRoot(fromVersion); + this.#assertVersionNotBeforeShallowRoot(toVersion); + if (fromVersion.compare(toVersion) === 0) + return []; + const restoreVersion = this.version(); + const currentToFromForward = this.#recordsInVersionRange(restoreVersion, fromVersion); + const currentToFromRetreat = this.#recordsInVersionRange(fromVersion, restoreVersion); + const forwardRecords = this.#recordsInVersionRange(fromVersion, toVersion); + const retreatRecords = this.#recordsInVersionRange(toVersion, fromVersion); + const currentToFromMoveMode = movableMoveTransitionMode(currentToFromRetreat, currentToFromForward, this.#movableMovePeers); + const fromToToMoveMode = movableMoveTransitionMode(retreatRecords, forwardRecords, this.#movableMovePeers); + const useIncrementalTransition = this.#canTransitionRecords([...currentToFromRetreat, ...currentToFromForward], currentToFromMoveMode) && + this.#canTransitionRecords([...retreatRecords, ...forwardRecords], fromToToMoveMode); + let materializedVersion = restoreVersion; + let transitionFailed = false; + try { + if (materializedVersion.compare(fromVersion) !== 0) { + if (useIncrementalTransition) { + this.#applyVersionTransition(currentToFromRetreat, currentToFromForward, fromVersion, undefined, currentToFromMoveMode); + } + else { + this.#rebuildFromHistory(fromVersion); + } + materializedVersion = fromVersion; + } + const changed = changedContainerIds([...forwardRecords, ...retreatRecords]); + let before = new Map(); + let calculated = new Map(); + let mapKeysAtFrom = new Map(); + let mapKeysAtTo = new Map(); + if (useIncrementalTransition || retreatRecords.length === 0) { + const recording = { + beforeValues: new Map(), + eventStates: new Map(), + }; + if (useIncrementalTransition) { + this.#applyVersionTransition(retreatRecords, forwardRecords, toVersion, recording, fromToToMoveMode); + } + else { + this.#applyRecords(forwardRecords, recording); + } + before = recording.beforeValues; + calculated = this.#recordedEventDiffs(recording, true); + } + else { + before = this.#captureContainerEventValues(changed); + mapKeysAtFrom = this.#captureMapKeys(changed); + this.#rebuildFromHistory(toVersion); + mapKeysAtTo = this.#captureMapKeys(changed); + } + materializedVersion = toVersion; + return [...changed] + .flatMap((id) => { + const container = this.#containers.get(id); + return container === undefined ? [] : [container]; + }) + .sort((left, right) => containerDepth(left) - containerDepth(right)) + .flatMap((container) => { + const diff = calculated.get(container.id) ?? + containerDiff(container, before.get(container.id), { + from: mapKeysAtFrom.get(container.id), + to: mapKeysAtTo.get(container.id), + }); + if (isEmptyContainerDiff(diff)) + return []; + return [ + [container.id, forJson ? diffForJson(diff) : diff], + ]; + }); + } + catch (error) { + transitionFailed = useIncrementalTransition; + throw error; + } + finally { + if (transitionFailed) { + this.#rebuildFromHistory(restoreVersion); + } + else if (materializedVersion.compare(restoreVersion) !== 0) { + if (useIncrementalTransition) { + if (materializedVersion.compare(toVersion) === 0) { + this.#applyVersionTransition(forwardRecords, retreatRecords, fromVersion, undefined, fromToToMoveMode); + materializedVersion = fromVersion; + } + if (materializedVersion.compare(restoreVersion) !== 0) { + this.#applyVersionTransition(currentToFromForward, currentToFromRetreat, restoreVersion, undefined, currentToFromMoveMode); + } + } + else { + this.#rebuildFromHistory(restoreVersion); + } + } + } + } + applyDiff(diffBatch) { + if (!Array.isArray(diffBatch)) + throw new TypeError("diff batch must be an array"); + if (this.#detached && !this.#detachedEditing) { + throw new Error("cannot edit a detached document; call attach() first"); + } + const containerRemap = new Map(); + const treeRemap = new Map(); + for (const entry of diffBatch) { + if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string") { + throw new TypeError("each diff entry must be a [ContainerID, Diff] tuple"); + } + const sourceId = entry[0]; + const diff = entry[1]; + const container = this.#resolveDiffContainer(sourceId, containerRemap); + if (container === undefined) + continue; + this.#applyContainerDiff(container, diff, containerRemap, treeRemap); + } + } + revertTo(frontiers) { + if (this.#detached && !this.#detachedEditing) { + throw new Error("cannot edit a detached document; call attach() first"); + } + this.#commit({}, true); + const diff = this.diff(this.frontiers(), frontiers, false); + this.applyDiff(diff); + } + #resolveDiffContainer(sourceId, remap) { + const mapped = remap.get(sourceId); + if (mapped !== undefined) + return mapped; + const parsed = parseContainerId(sourceId); + if (parsed.kind === "root" && !isMergeableContainerId(parsed)) { + return this.#getOrCreateContainer(parsed); + } + const existing = this.#containers.get(formatContainerId(parsed)); + return existing !== undefined && !this._isContainerDeleted(existing) + ? existing + : undefined; + } + #applyContainerDiff(container, diff, containerRemap, treeRemap) { + if (diff.type === "map") { + if (!(container instanceof LoroMap)) + throw diffKindMismatch(container, diff.type); + for (const [key, value] of Object.entries(diff.updated)) { + if (value === undefined) { + container.delete(key); + continue; + } + const sourceChildId = diffContainerId(value); + if (sourceChildId === undefined) { + container.set(key, value); + } + else { + this.#applyMapChildDiff(container, key, sourceChildId, containerRemap); + } + } + return; + } + if (diff.type === "text") { + if (!(container instanceof LoroText)) + throw diffKindMismatch(container, diff.type); + container.applyDelta(diff.diff); + return; + } + if (diff.type === "counter") { + if (!(container instanceof LoroCounter)) + throw diffKindMismatch(container, diff.type); + container.increment(diff.increment); + return; + } + if (diff.type === "tree") { + if (!(container instanceof LoroTree)) + throw diffKindMismatch(container, diff.type); + this.#applyTreeDiff(container, diff.diff, containerRemap, treeRemap); + return; + } + if (!(container instanceof LoroList)) + throw diffKindMismatch(container, diff.type); + let position = 0; + for (const operation of diff.diff) { + if ("retain" in operation) { + position += operation.retain; + } + else if ("delete" in operation) { + container.delete(position, operation.delete); + } + else { + for (const value of operation.insert) { + const sourceChildId = diffContainerId(value); + if (sourceChildId === undefined) { + container.insert(position, value); + } + else { + const parsed = parseContainerId(sourceChildId); + if (parsed.kind === "root") { + throw new TypeError("a root container cannot be inserted as a child"); + } + const child = createContainer(codecTypeToPublic(parsed.containerType)); + const attached = container.insertContainer(position, child); + containerRemap.set(sourceChildId, attached); + } + position += 1; + } + } + } + } + #applyMapChildDiff(parent, key, sourceId, remap) { + const mapped = remap.get(sourceId); + const current = parent.get(key); + if (mapped !== undefined && current === mapped) + return; + const parsed = parseContainerId(sourceId); + const type = codecTypeToPublic(parsed.containerType); + if (isMergeableContainerId(parsed)) { + const child = ensureMergeableChild(parent, key, type); + remap.set(sourceId, child); + return; + } + if (parsed.kind === "root") { + throw new TypeError("a root container cannot be assigned as a child"); + } + if (isContainer(current) && current.id === sourceId) { + remap.set(sourceId, current); + return; + } + const child = createContainer(type); + remap.set(sourceId, parent.setContainer(key, child)); + } + #applyTreeDiff(tree, diff, containerRemap, treeRemap) { + const resolveNode = (source) => treeRemap.get(source) ?? (tree.has(source) ? source : undefined); + for (const item of diff) { + if (item.action !== "delete") + continue; + const target = resolveNode(item.target); + if (target !== undefined) + tree.delete(target); + } + const pending = diff.filter((item) => item.action !== "delete"); + while (pending.length > 0) { + let progressed = false; + for (let index = 0; index < pending.length;) { + const item = pending[index]; + const parent = item.parent === undefined ? undefined : resolveNode(item.parent); + if (item.parent !== undefined && parent === undefined) { + index += 1; + continue; + } + if (item.action === "create") { + const node = tree.createNode(parent, item.index); + treeRemap.set(item.target, node.id); + const sourceNode = parseTreeId(item.target); + const sourceMetaId = formatContainerId({ + kind: "normal", + ...sourceNode, + containerType: CodecContainerType.Map, + }); + containerRemap.set(sourceMetaId, node.data); + } + else { + const target = resolveNode(item.target); + if (target !== undefined) + tree.move(target, parent, item.index); + } + pending.splice(index, 1); + progressed = true; + } + if (!progressed) { + throw new RangeError("tree diff refers to a parent that does not exist"); + } + } + } + getDeepValueWithId() { + return Object.fromEntries([...this.#roots].map(([name, container]) => [ + name, + containerDeepValueWithId(container), + ])); + } + getShallowValue() { + return Object.fromEntries([...this.#roots].map(([name, container]) => [name, container.id])); + } + getDeepValueWithID() { + return this.getDeepValueWithId(); + } + version() { + const version = this.#checkoutVersion?.clone() ?? this.#historyVersion(); + if (this.#pending !== undefined) { + const end = this.#pending.id.counter + this.#pending.operationLength; + if (end > (version.get(this.#peer) ?? 0)) + version.set(this.#peer, end); + } + return version; + } + oplogVersion() { + return this.#historyVersion(); + } + #historyVersion() { + if (this.#deferredSnapshotHistory !== undefined) { + return this.#deferredSnapshotHistory.endVersion.clone(); + } + const version = this.#shallowStartVersion.clone(); + for (const [peer, end] of this.#historyEndByPeer) { + if (end > (version.get(peer) ?? 0)) + version.set(peer, end); + } + return version; + } + frontiers() { + return this.#frontiersCodec().map(formatOpId); + } + oplogFrontiers() { + if (this.#deferredSnapshotHistory !== undefined) { + return [...this.#deferredSnapshotHistory.frontiers] + .sort(compareIds) + .map(formatOpId); + } + return [...this.#historyFrontiers.values()].sort(compareIds).map(formatOpId); + } + frontiersToVV(frontiers) { + const version = new VersionVector(); + for (const [peer, counter] of this.#causalVersionAt(frontiers.map(parseOpId))) { + version.set(peer, counter); + } + return version; + } + #versionForExistingFrontiers(frontiers) { + for (const frontier of frontiers) { + if (this.#recordContaining(parseOpId(frontier)) === undefined) { + throw new RangeError(`frontiers include unknown id ${frontier.counter}@${frontier.peer}`); + } + } + return this.frontiersToVV(frontiers); + } + vvToFrontiers(version) { + return this.#frontiersForVersion(version).map(formatOpId); + } + cmpWithFrontiers(frontiers) { + const current = this.oplogFrontiers(); + if (frontierSetsEqual(current, frontiers)) + return 0; + const version = this.#historyVersion(); + return frontiers.every((frontier) => { + const id = parseOpId(frontier); + return id.counter < (version.get(id.peer) ?? 0); + }) + ? 1 + : -1; + } + cmpFrontiers(left, right) { + return this.#versionForExistingFrontiers(left).compare(this.#versionForExistingFrontiers(right)); + } + changeCount() { + this.#materializeDeferredHistory(); + return this.#history.size; + } + debugHistory() { + this.#sortedHistory(); + } + opCount() { + return this.#deferredSnapshotHistory?.operationCount ?? this.#historyOperationCount; + } + getAllChanges() { + const changes = new Map(); + for (const { change } of this.#sortedHistory()) { + const peer = peerIdToString(change.id.peer); + const peerChanges = changes.get(peer); + if (peerChanges === undefined) + changes.set(peer, [publicChange(change)]); + else + peerChanges.push(publicChange(change)); + } + return changes; + } + getChangeAt(id) { + const target = parseOpId(id); + const record = this.#recordContaining(target); + if (record === undefined) { + throw new RangeError(`change ${target.counter}@${target.peer} is unknown`); + } + return publicChange(record.change); + } + getChangeAtLamport(peer, lamport) { + this.#materializeDeferredHistory(); + const parsedPeer = parsePeerId(peer); + const records = this.#historyByPeer.get(parsedPeer) ?? []; + let low = 0; + let high = records.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (records[middle].change.lamport <= lamport) + low = middle + 1; + else + high = middle; + } + const record = low > 0 ? records[low - 1] : undefined; + return record === undefined ? undefined : publicChange(record.change); + } + getOpsInChange(id) { + this.#commit({}, true); + const record = this.#recordContaining(parseOpId(id)); + if (record === undefined) { + throw new RangeError(`change ${id.counter}@${id.peer} is unknown`); + } + return historyRecordToJsonChange(record, undefined, false).ops; + } + getUncommittedOpsAsJson() { + const pending = this.#pending; + if (pending === undefined || pending.operations.length === 0) + return undefined; + const timestamp = this.#nextCommitOptions.timestamp ?? + (this.#recordTimestamp ? Date.now() / 1000 : 0); + const record = { + change: { + id: pending.id, + dependencies: pending.dependencies, + lamport: pending.lamport, + timestamp: numberToI64(timestamp), + message: this.#nextCommitOptions.message, + operations: pending.operations, + }, + keys: pending.keys, + keyIndices: undefined, + }; + return historyRecordsToJsonSchema([record], this.#historyVersion(), false); + } + travelChangeAncestors(ids, callback) { + this.#commit({}, true); + const visited = new Set(); + let stopped = false; + const visit = (id) => { + if (stopped) + return; + const record = this.#recordContaining(id); + if (record === undefined) { + if (id.counter < (this.#shallowStartVersion.get(id.peer) ?? 0)) + return; + throw new RangeError(`change ${id.counter}@${id.peer} is unknown`); + } + const key = changeKey(record.change.id); + if (visited.has(key)) + return; + for (const dependency of record.change.dependencies) + visit(dependency); + if (stopped || visited.has(key)) + return; + visited.add(key); + if (callback(publicChange(record.change)) === false) + stopped = true; + }; + for (const id of ids.map(parseOpId)) + visit(id); + } + findIdSpansBetween(from, to) { + const fromVersion = this.#versionForExistingFrontiers(from); + const toVersion = this.#versionForExistingFrontiers(to); + this.#assertVersionNotBeforeShallowRoot(fromVersion); + this.#assertVersionNotBeforeShallowRoot(toVersion); + const retreat = []; + const forward = []; + const peers = new Set([ + ...fromVersion._codecEntriesUnsorted().map(({ peer }) => peer), + ...toVersion._codecEntriesUnsorted().map(({ peer }) => peer), + ]); + for (const peer of [...peers].sort((left, right) => left < right ? -1 : left > right ? 1 : 0)) { + const fromCounter = fromVersion.get(peer) ?? 0; + const toCounter = toVersion.get(peer) ?? 0; + if (fromCounter < toCounter) { + forward.push({ + peer: peerIdToString(peer), + counter: fromCounter, + length: toCounter - fromCounter, + }); + } + else if (fromCounter > toCounter) { + retreat.push({ + peer: peerIdToString(peer), + counter: toCounter, + length: fromCounter - toCounter, + }); + } + } + return { retreat, forward }; + } + getChangedContainersIn(id, len) { + this.#commit({}, true); + this.#materializeDeferredHistory(); + if (!Number.isSafeInteger(len) || len < 0) { + throw new RangeError(`change range length is out of range: ${len}`); + } + const start = parseOpId(id); + const end = start.counter + len; + const containers = new Set(); + const records = this.#historyByPeer.get(start.peer) ?? []; + let recordIndex = Math.max(0, lowerBoundHistory(records, start.counter + 1) - 1); + for (; recordIndex < records.length; recordIndex += 1) { + const { change } = records[recordIndex]; + if (change.id.counter >= end) + break; + let operationIndex = lowerBoundOperation(change.operations, start.counter + 1) - 1; + operationIndex = Math.max(0, operationIndex); + for (; operationIndex < change.operations.length; operationIndex += 1) { + const operation = change.operations[operationIndex]; + if (operation.counter >= end) + break; + if (operation.counter < end && + operation.counter + operation.length > start.counter) { + containers.add(formatContainerId(operation.container)); + } + } + } + return [...containers]; + } + #recordContaining(id) { + this.#materializeDeferredHistory(); + const records = this.#historyByPeer.get(id.peer); + if (records === undefined) + return undefined; + const index = lowerBoundHistory(records, id.counter + 1) - 1; + const record = index >= 0 ? records[index] : undefined; + return record !== undefined && + id.counter < record.change.id.counter + changeLength(record.change) + ? record + : undefined; + } + #greatestTimestampAt(frontiers) { + const version = this.#causalVersionAt(frontiers); + let greatest = 0n; + for (const [peer, counter] of version) { + if (counter === 0) + continue; + const record = this.#recordContaining({ peer, counter: counter - 1 }); + if (record !== undefined) + greatest = maxBigInt(greatest, record.change.timestamp); + } + return greatest; + } + #mergeablePreviousRecord(change, interval) { + if (change.id.counter === 0) + return undefined; + const previous = this.#recordContaining({ + peer: change.id.peer, + counter: change.id.counter - 1, + }); + return previous !== undefined && canMergeChanges(previous.change, change, interval) + ? previous + : undefined; + } + fork() { + return this.forkAt(this.frontiers()); + } + forkAt(frontiers) { + this.#commit({}, true); + const version = this.#versionForExistingFrontiers(frontiers); + this.#assertVersionNotBeforeShallowRoot(version); + const fork = new LoroDoc(); + fork.#recordTimestamp = this.#recordTimestamp; + fork.#changeMergeInterval = this.#changeMergeInterval; + fork.#detachedEditing = this.#detachedEditing; + fork.#hideEmptyRoots = this.#hideEmptyRoots; + fork.#textStyles = new Map(this.#textStyles); + fork.#defaultTextStyle = this.#defaultTextStyle; + fork.#shallowStartVersion = this.#shallowStartVersion.clone(); + fork.#shallowRootVersion = this.#shallowRootVersion.clone(); + fork.#shallowRootFrontiers = this.#shallowRootFrontiers.map((id) => ({ ...id })); + fork.#shallowRootStore = this.#shallowRootStore; + fork.#integrateHistory(this.#recordsAtVersion(version), undefined); + fork.#rebuildFromHistory(); + return fork; + } + attach() { + this.checkoutToLatest(); + } + checkout(frontiers) { + this.#commit({}, true); + for (const frontier of frontiers) { + if (this.#recordContaining(parseOpId(frontier)) === undefined) { + throw new RangeError(`frontier ${frontier.counter}@${frontier.peer} is unknown`); + } + } + const before = this.#hasEventSubscribers() ? this.#frontiersCodec() : []; + const currentVersion = this.version(); + const targetVersion = this.frontiersToVV(frontiers); + this.#assertVersionNotBeforeShallowRoot(targetVersion); + const forwardRecords = this.#recordsInVersionRange(currentVersion, targetVersion); + const retreatRecords = this.#recordsInVersionRange(targetVersion, currentVersion); + const changed = this.#hasEventSubscribers() + ? changedContainerIds([...forwardRecords, ...retreatRecords]) + : new Set(); + let beforeValues = new Map(); + let preparedDiffs = new Map(); + this.#checkoutVersion = targetVersion; + this.#detached = true; + const changedRecords = [...forwardRecords, ...retreatRecords]; + const movableMoveMode = movableMoveTransitionMode(retreatRecords, forwardRecords, this.#movableMovePeers); + if (this.#canTransitionRecords(changedRecords, movableMoveMode)) { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + this.#applyVersionTransition(retreatRecords, forwardRecords, targetVersion, recording, movableMoveMode); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + else if (retreatRecords.length === 0 && + !hasMaterializedSequenceInsertions(forwardRecords, this.#containers)) { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + this.#applyRecords(forwardRecords, recording); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + else { + if (this.#hasEventSubscribers()) { + beforeValues = this.#captureContainerEventValues(changed); + } + this.#rebuildFromHistory(targetVersion); + } + this.#emit("checkout", undefined, before, changed.size > 0 ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); + if (this.#detachedEditing) + this.#renewPeerId(); + } + detach() { + this.#commit({}, true); + this.#checkoutVersion = this.version(); + this.#detached = true; + } + isDetached() { + return this.#detached; + } + checkoutToLatest() { + this.#commit({}, true); + const wasDetached = this.#detached; + // Without subscribers (or when already attached) no event is emitted, so + // skip building the frontier array and the changed-container set. + const emitting = wasDetached && this.#hasEventSubscribers(); + const before = emitting ? this.#frontiersCodec() : []; + const currentVersion = this.version(); + const latestVersion = this.#historyVersion(); + const forwardRecords = this.#recordsInVersionRange(currentVersion, latestVersion); + const retreatRecords = this.#recordsInVersionRange(latestVersion, currentVersion); + const changed = emitting + ? changedContainerIds([...forwardRecords, ...retreatRecords]) + : new Set(); + let beforeValues = new Map(); + let preparedDiffs = new Map(); + this.#checkoutVersion = undefined; + this.#detached = false; + if (wasDetached) { + const changedRecords = [...forwardRecords, ...retreatRecords]; + const movableMoveMode = movableMoveTransitionMode(retreatRecords, forwardRecords, this.#movableMovePeers); + if (this.#canTransitionRecords(changedRecords, movableMoveMode)) { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + this.#applyVersionTransition(retreatRecords, forwardRecords, latestVersion, recording, movableMoveMode); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + else if (retreatRecords.length === 0 && + !hasMaterializedSequenceInsertions(forwardRecords, this.#containers)) { + const recording = this.#hasEventSubscribers() + ? { beforeValues: new Map(), eventStates: new Map() } + : undefined; + this.#applyRecords(forwardRecords, recording); + if (recording !== undefined) { + beforeValues = recording.beforeValues; + preparedDiffs = this.#recordedEventDiffs(recording); + } + } + else { + if (this.#hasEventSubscribers()) { + beforeValues = this.#captureContainerEventValues(changed); + } + this.#rebuildFromHistory(); + } + this.#emit("checkout", undefined, before, changed.size > 0 ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); + if (this.#detachedEditing) + this.#renewPeerId(); + } + } + #renewPeerId() { + let peer = generatePeerId(); + while (peer === this.#peer || peer === 0xffffffffffffffffn) { + peer = generatePeerId(); + } + this.#peer = peer; + this.#nextCounter = this.oplogVersion().get(peer) ?? 0; + } + isShallow() { + return this.#shallowRootStore !== undefined; + } + shallowSinceVV() { + return this.#shallowStartVersion.clone(); + } + shallowSinceFrontiers() { + return this.#shallowRootFrontiers.map(formatOpId); + } + subscribe(listener) { + this.#subscribers.add(listener); + return () => this.#subscribers.delete(listener); + } + subscribeLocalUpdates(listener) { + this.#localUpdateSubscribers.add(listener); + return () => this.#localUpdateSubscribers.delete(listener); + } + subscribeFirstCommitFromPeer(listener) { + this.#firstCommitSubscribers.add(listener); + return () => this.#firstCommitSubscribers.delete(listener); + } + subscribePreCommit(listener) { + this.#preCommitSubscribers.add(listener); + return () => this.#preCommitSubscribers.delete(listener); + } + subscribeJsonpath(path, callback) { + const matches = compileJsonPathEventMatcher(path); + return this.subscribe((event) => { + if (event.events.some(matches)) + callback(); + }); + } + JSONPath(path) { + if (path === "$") + return [Object.fromEntries(this.#roots)]; + return evaluateJsonPath(this.#roots, path); + } + getPathToContainer(id) { + const container = this.getContainerById(id); + if (container === undefined || this._isContainerDeleted(container)) + return undefined; + return containerPath(container); + } + getByPath(path) { + const parts = path.split("/").filter(Boolean); + if (parts.length === 0) + return undefined; + let value = this.#roots.get(parts[0]); + for (const part of parts.slice(1)) { + if (value instanceof LoroMap) { + value = value.get(part); + } + else if (value instanceof LoroList) { + value = value.get(parsePathIndex(part)); + } + else if (value instanceof LoroTree) { + value = treeNodeAtPath(value, undefined, part); + } + else if (value instanceof LoroTreeNode) { + const index = parseOptionalPathIndex(part); + value = index === undefined ? value.data.get(part) : value._childAt(index); + } + else if (Array.isArray(value)) { + value = value[parsePathIndex(part)]; + } + else if (typeof value === "object" && value !== null) { + value = value[part]; + } + else { + return undefined; + } + if (value === undefined) + return undefined; + } + return value instanceof LoroTreeNode ? value.data : value; + } + getCursorPos(cursor) { + const container = this.getContainerById(cursor.containerId()); + if (container === undefined || this._isContainerDeleted(container)) + return undefined; + const id = cursor._idValue(); + if (!(container instanceof LoroList || container instanceof LoroText)) + return undefined; + const publicLength = container.length; + if (id === undefined) { + return { + offset: Math.min(cursor._originPositionValue(), publicLength), + side: cursor.side(), + }; + } + const target = container._sequence.findById(id); + if (target === undefined) + return undefined; + const publicOffset = (element) => container instanceof LoroText + ? (container._sequence.visibleMetricOffsetOf(element, "utf16") ?? + 0) + : (container._sequence.visibleIndexOf(element) ?? 0); + if (!target.deleted) { + const offset = publicOffset(target); + return { + offset: offset + + (cursor.side() === 1 + ? container instanceof LoroText + ? target.value.length + : 1 + : 0), + side: cursor.side(), + }; + } + const next = container._sequence.nextVisible(target); + if (next !== undefined) { + const offset = publicOffset(next); + return { + offset, + side: -1, + update: new Cursor(container.id, next.id, -1, offset), + }; + } + const previous = container._sequence.previousVisible(target); + if (previous !== undefined) { + return { + offset: publicLength, + side: 1, + update: new Cursor(container.id, previous.id, 1, publicLength), + }; + } + return { + offset: 0, + side: 0, + update: new Cursor(container.id, undefined, 0, 0), + }; + } + _subscribeContainer(container, listener) { + let listeners = this.#containerSubscribers.get(container.id); + if (listeners === undefined) { + listeners = new Set(); + this.#containerSubscribers.set(container.id, listeners); + } + listeners.add(listener); + return () => { + listeners.delete(listener); + if (listeners.size === 0 && + this.#containerSubscribers.get(container.id) === listeners) { + this.#containerSubscribers.delete(container.id); + } + }; + } + _isContainerDeleted(container) { + if (container._codecId?.kind === "root" && + !isMergeableContainerId(container._codecId)) + return !this.#roots.has(container._codecId.name); + const parent = container.parent(); + if (container._codecId !== undefined && + isMergeableContainerId(container._codecId) && + parent === undefined) + return true; + const binding = container._parentLink?.binding ?? + (parent === undefined ? undefined : recoverParentBinding(container, parent)); + if (binding?.kind === "map" && parent instanceof LoroMap) { + const record = parent._entries.get(binding.key); + return record === undefined || record.deleted || record.value !== container; + } + if (binding?.kind === "sequence" && parent instanceof LoroList) { + return binding.element.deleted || binding.element.value !== container; + } + if (binding?.kind === "tree" && parent instanceof LoroTree) { + return parent._isRecordDeleted(binding.record) || binding.record.data !== container; + } + return parent !== undefined; + } + _undoIdSpan(peer, range) { + this.#commit({}, true); + this.#materializeDeferredHistory(); + const parsedPeer = parsePeerId(peer); + const records = this.#historyByPeer.get(parsedPeer) ?? []; + let recordIndex = Math.min(records.length - 1, lowerBoundHistory(records, range.end) - 1); + for (; recordIndex >= 0; recordIndex -= 1) { + const record = records[recordIndex]; + const changeEnd = record.change.id.counter + changeLength(record.change); + if (changeEnd <= range.start) + break; + let operationIndex = Math.min(record.change.operations.length - 1, lowerBoundOperation(record.change.operations, range.end) - 1); + for (; operationIndex >= 0; operationIndex -= 1) { + const operation = record.change.operations[operationIndex]; + if (operation.counter + operation.length <= range.start) + break; + this.#undoOperation(record, operation, parsedPeer, range); + } + } + } + _transformUndoCursors(cursors) { + return cursors.map((cursor) => { + const container = this.getContainerById(cursor.containerId()); + if (!(container instanceof LoroText || container instanceof LoroList)) { + return cursor; + } + const position = Math.min(cursor._originPositionValue(), container.length); + return container.getCursor(position, cursor.side()) ?? cursor; + }); + } + #undoOperation(record, operation, peer, range) { + const container = this.#containers.get(formatContainerId(operation.container)); + if (container === undefined) + return; + const content = operation.content; + const overlapStart = Math.max(operation.counter, range.start); + const overlapEnd = Math.min(operation.counter + operation.length, range.end); + if (overlapStart >= overlapEnd) + return; + switch (content.type) { + case "text-insert": + if (container instanceof LoroText) { + this.#deleteInsertedElements(container, peer, overlapStart, overlapEnd); + } + return; + case "list-insert": + case "movable-list-insert": + if (container instanceof LoroList) { + this.#deleteInsertedElements(container, peer, overlapStart, overlapEnd); + } + return; + case "text-delete": + if (container instanceof LoroText) { + this.#restoreDeletedElements(container, peer, range, overlapStart, overlapEnd); + } + return; + case "list-delete": + case "movable-list-delete": + if (container instanceof LoroList) { + this.#restoreDeletedElements(container, peer, range, overlapStart, overlapEnd); + } + return; + case "map-insert": + case "map-delete": + if (container instanceof LoroMap) { + this.#undoMapOperation(container, record, operation); + } + return; + case "tree-create": + case "tree-move": + case "tree-delete": + if (container instanceof LoroTree) { + this.#undoTreeOperation(container, record, operation); + } + return; + case "future": + if (container instanceof LoroCounter && + (content.value.type === "double" || content.value.type === "i64")) { + container.increment(-(content.value.type === "double" + ? content.value.value + : Number(content.value.value))); + } + else if (container instanceof LoroCounter && + content.value.type === "delta-int") { + container.increment(-content.value.value); + } + return; + case "text-mark": + case "text-mark-end": + case "movable-list-move": + case "movable-list-set": + return; + } + } + #deleteInsertedElements(container, peer, start, end) { + const positions = []; + for (let counter = start; counter < end; counter += 1) { + const element = container._sequence.findById({ peer, counter }); + if (element === undefined || element.deleted) + continue; + const index = container._sequence.visibleIndexOf(element); + if (index !== undefined) + positions.push(index); + } + positions.sort((left, right) => left - right); + const ranges = []; + for (const position of positions) { + const previous = ranges.at(-1); + if (previous !== undefined && previous.start + previous.length === position) { + previous.length += 1; + } + else { + ranges.push({ start: position, length: 1 }); + } + } + for (const selected of ranges.reverse()) { + if (container instanceof LoroText) { + this._textDelete(container, selected.start, selected.length); + } + else { + this._sequenceDelete(container, selected.start, selected.length); + } + } + } + #restoreDeletedElements(container, peer, range, overlapStart, overlapEnd) { + const belongsToUndo = (id) => id.peer === peer && id.counter >= range.start && id.counter < range.end; + const belongsToOperation = (id) => id.peer === peer && id.counter >= overlapStart && id.counter < overlapEnd; + const targets = container._sequence + .elementsDeletedBy(peer, overlapStart, overlapEnd) + .filter((element) => !belongsToUndo(element.id) && + container._sequence.someDeletion(element, belongsToOperation) && + container._sequence.everyDeletion(element, belongsToUndo)); + const groups = []; + for (const target of targets) { + const previous = groups.at(-1)?.at(-1); + if (previous !== undefined && + container._sequence.physicalIndexOf(previous) + 1 === + container._sequence.physicalIndexOf(target)) { + groups.at(-1).push(target); + } + else { + groups.push([target]); + } + } + for (const group of groups) { + const position = container._sequence.visibleIndexOf(group[0]); + if (container instanceof LoroText) { + const text = group + .map((element) => { + if (typeof element.value !== "string") { + throw new TypeError("text element value must be a string"); + } + return element.value; + }) + .join(""); + this._textInsert(container, position, text); + } + else { + let offset = 0; + for (const element of group) { + if (isContainer(element.value)) { + const child = createContainer(element.value.kind()); + restoreBlueprint(child, captureBlueprint(element.value)); + this._listInsertContainer(container, position + offset, child); + } + else { + this._listInsert(container, position + offset, cloneRuntimeValue(element.value)); + } + offset += 1; + } + } + } + } + #undoMapOperation(map, record, operation) { + const content = operation.content; + if (content.type !== "map-insert" && content.type !== "map-delete") + return; + const writer = operationWriter(record.change, operation); + const current = map._entries.get(content.key); + if (current === undefined || compareWriter(current.writer, writer) !== 0) + return; + const previous = this.#previousMapOperation(operation.container, content.key, writer); + if (previous === undefined || previous.operation.content.type === "map-delete") { + map.delete(content.key); + return; + } + const previousContent = previous.operation.content; + if (previousContent.type !== "map-insert") + return; + const previousId = { + peer: previous.record.change.id.peer, + counter: previous.operation.counter, + }; + const rawValue = this.#decodeRuntimeValue(previousContent.value, previous.record.keys, previousId, map); + const value = this.#materializeMapValue(map, content.key, rawValue); + if (isContainer(value)) { + const child = createContainer(value.kind()); + restoreBlueprint(child, captureBlueprint(value)); + map.setContainer(content.key, child); + } + else { + map.set(content.key, cloneRuntimeValue(value)); + } + } + #previousMapOperation(containerId, key, before) { + const operations = this.#mapOperationHistory + .get(formatContainerId(containerId)) + ?.get(key); + if (operations === undefined) + return undefined; + const previous = operations.byWriter.at(operations.byWriter._lowerBoundBy((operation) => compareWriter(operation.writer, before)) - 1); + return previous === undefined + ? undefined + : { record: previous.record, operation: previous.operation }; + } + #undoTreeOperation(tree, record, operation) { + const content = operation.content; + if (content.type !== "tree-create" && + content.type !== "tree-move" && + content.type !== "tree-delete") { + return; + } + const writer = operationWriter(record.change, operation); + const node = tree._nodes.get(formatTreeId(content.subject)); + if (node === undefined || compareWriter(node.writer, writer) !== 0) + return; + const previous = this.#previousTreeOperation(operation.container, content.subject, writer); + if (previous === undefined || previous.operation.content.type === "tree-delete") { + this.#appendAndApply(tree, { type: "tree-delete", subject: content.subject }, 1); + return; + } + const previousContent = previous.operation.content; + if (previousContent.type !== "tree-create" && previousContent.type !== "tree-move") { + return; + } + this.#appendAndApply(tree, { + type: "tree-move", + subject: content.subject, + parent: previousContent.parent, + position: previousContent.position.slice(), + }, 1); + } + #previousTreeOperation(containerId, subject, before) { + const operations = this.#treeOperationHistory + .get(formatContainerId(containerId)) + ?.get(idKey(subject)); + if (operations === undefined) + return undefined; + const previous = operations.byWriter.at(operations.byWriter._lowerBoundBy((operation) => compareWriter(operation.writer, before)) - 1); + return previous === undefined + ? undefined + : { record: previous.record, operation: previous.operation }; + } + _mapSet(container, key, value) { + const current = container._entries.get(key); + if (current !== undefined && + !current.deleted && + eventValuesEqual(current.value, value)) { + return; + } + const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); + this.#appendAndApply(container, { type: "map-insert", key, value: encoded }, 1); + } + _mapDelete(container, key) { + const current = container._entries.get(key); + if (current === undefined || current.deleted) + return; + this.#appendAndApply(container, { type: "map-delete", key }, 1); + } + _mapSetContainer(container, key, child) { + return this.#attachChild(container, child, (rawType) => ({ + type: "map-insert", + key, + value: { type: "container-type", value: rawType }, + })); + } + _mapEnsureMergeable(container, key, type) { + const parentId = container._codecId; + if (parentId === undefined || container._doc !== this) { + throw new Error("cannot ensure a mergeable child on a detached map"); + } + const codecType = publicTypeToCodec(type); + const childId = newMergeableContainerId(parentId, key, codecType); + const marker = mergeableMarker(parentId, key, codecType); + const existing = container._entries.get(key); + if (existing !== undefined && !existing.deleted && existing.rawValue !== null) { + const existingType = parseMergeableMarker(parentId, key, existing.rawValue); + if (existingType === undefined) { + throw new TypeError(`cannot create a mergeable ${type} at key ${JSON.stringify(key)}: ` + + "the key already holds a non-mergeable value"); + } + if (existing.rawValue instanceof Uint8Array && + bytesEqual(existing.rawValue, marker)) { + return this.#getOrCreateContainer(childId, container); + } + } + const child = this.#getOrCreateContainer(childId, container); + this.#appendAndApply(container, { type: "map-insert", key, value: { type: "binary", value: marker } }, 1); + return child; + } + _listInsert(container, position, value) { + const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); + const type = container instanceof LoroMovableList ? "movable-list-insert" : "list-insert"; + this.#appendAndApply(container, { type, position, values: [encoded] }, 1); + } + _listInsertContainer(container, position, child) { + return this.#attachChild(container, child, (rawType) => ({ + type: container instanceof LoroMovableList ? "movable-list-insert" : "list-insert", + position, + values: [{ type: "container-type", value: rawType }], + })); + } + _sequenceDelete(container, position, length) { + this.#deleteSequenceRuns(container, position, length, container instanceof LoroMovableList ? "movable-list-delete" : "list-delete"); + } + _textInsert(container, position, text) { + this.#appendAndApply(container, { type: "text-insert", position, value: text }, unicodeScalarLength(text)); + } + _textDelete(container, position, length) { + this.#deleteTextRuns(container, position, length); + } + _textMark(container, start, end, key, value) { + const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); + const info = textStyleInfoByte(this.#textStyleExpand(key), value === null); + this.#appendAndApply(container, { type: "text-mark", start, end, key, value: encoded, info }, 1); + this.#appendAndApply(container, { type: "text-mark-end" }, 1); + } + _movableMove(container, from, to) { + const element = container._visibleElementAt(from); + this.#appendAndApply(container, { + type: "movable-list-move", + from, + to, + elementId: { peer: element.id.peer, lamport: element.lamport }, + }, 1); + } + _movableSet(container, position, value) { + const element = container._visibleElementAt(position); + if (eventValuesEqual(element.value, value)) + return; + const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); + this.#appendAndApply(container, { + type: "movable-list-set", + elementId: { peer: element.id.peer, lamport: element.lamport }, + value: encoded, + }, 1); + } + _movableSetContainer(container, position, child) { + const element = container._visibleElementAt(position); + return this.#attachChild(container, child, (rawType) => ({ + type: "movable-list-set", + elementId: { peer: element.id.peer, lamport: element.lamport }, + value: { type: "container-type", value: rawType }, + })); + } + _counterIncrement(container, value) { + if (value === 0) + return; + this.#appendAndApply(container, { type: "future", property: 0, value: { type: "double", value } }, 1); + } + #textStyleExpand(key) { + const baseKey = key.split(":", 1)[0]; + const configured = this.#textStyles.get(baseKey) ?? this.#defaultTextStyle; + if (configured === undefined) { + throw new RangeError(`text style ${JSON.stringify(baseKey)} is not configured`); + } + return configured; + } + _treeCreate(tree, parent, index) { + const pending = this.#ensurePending(); + const subject = { peer: this.#peer, counter: this.#nextOperationCounter(pending) }; + const parentId = parent === undefined ? undefined : parseTreeId(parent); + if (parent !== undefined && !tree.has(parent)) { + throw new RangeError(`tree parent ${parent} does not exist`); + } + const position = tree._positionFor(parentId, index); + this.#appendAndApply(tree, { type: "tree-create", subject, parent: parentId, position }, 1); + return new LoroTreeNode(tree, subject); + } + _treeMove(tree, target, parent, index) { + if (!tree.has(target)) + throw new RangeError(`tree node ${target} does not exist`); + if (parent !== undefined && !tree.has(parent)) { + throw new RangeError(`tree parent ${parent} does not exist`); + } + const subject = parseTreeId(target); + const parentId = parent === undefined ? undefined : parseTreeId(parent); + let ancestor = parentId; + while (ancestor !== undefined) { + if (idsEqual(ancestor, subject)) { + throw new RangeError("cannot move a tree node below itself or its descendant"); + } + ancestor = tree._nodes.get(formatTreeId(ancestor))?.parent; + } + const position = tree._positionFor(parentId, index, subject); + this.#appendAndApply(tree, { type: "tree-move", subject, parent: parentId, position }, 1); + } + _treeDelete(tree, target) { + this.#appendAndApply(tree, { type: "tree-delete", subject: parseTreeId(target) }, 1); + } + #getContainer(nameOrId, expectedType) { + const byId = isContainerId(nameOrId); + const id = byId + ? parseContainerId(nameOrId) + : { kind: "root", name: nameOrId, containerType: publicTypeToCodec(expectedType) }; + if (codecTypeToPublic(id.containerType) !== expectedType) { + throw new TypeError(`container ${nameOrId} is not a ${expectedType}`); + } + if (byId && + (id.kind === "normal" || isMergeableContainerId(id)) && + !this.#containers.has(formatContainerId(id))) { + throw new RangeError(`container ${nameOrId} does not exist in this document`); + } + const container = this.#getOrCreateContainer(id); + if (id.kind === "root" && !isMergeableContainerId(id)) { + this.#roots.set(id.name, container); + } + return container; + } + #getOrCreateContainer(id, parent) { + const key = this.#containerKey(id); + const existing = this.#containers.get(key); + if (existing !== undefined) { + if (parent !== undefined) + existing._parentLink = { container: parent }; + return existing; + } + const container = createContainer(codecTypeToPublic(id.containerType)); + container._attach(this, id, parent); + this.#containers.set(key, container); + if (id.kind === "root" && !isMergeableContainerId(id)) { + this.#roots.set(id.name, container); + } + return container; + } + #containerKey(id) { + let key = this.#containerKeys.get(id); + if (key === undefined) { + key = formatContainerId(id); + this.#containerKeys.set(id, key); + } + return key; + } + #ensurePending() { + if (this.#pending !== undefined) + return this.#pending; + this.#materializeDeferredHistory(); + if (this.#detached && !this.#detachedEditing) { + throw new Error("cannot edit a detached document; call attach() first"); + } + const from = this.#frontiersCodec(); + const lamport = from.reduce((max, dependency) => Math.max(max, this.#lamportAt(dependency) + 1), 0); + this.#nextCounter = Math.max(this.#nextCounter, this.oplogVersion().get(this.#peer) ?? 0); + this.#pending = { + id: { peer: this.#peer, counter: this.#nextCounter }, + dependencies: from.map((id) => ({ ...id })), + lamport, + from, + operations: [], + operationLength: 0, + causalVersion: this.#causalVersionAt(from), + keys: [], + keyIndices: new Map(), + changedContainers: new Set(), + beforeValues: new Map(), + eventStates: new Map(), + }; + return this.#pending; + } + #nextOperationCounter(pending) { + return pending.id.counter + pending.operationLength; + } + #appendAndApply(container, content, length, preferredChild) { + if (container._codecId === undefined || container._doc !== this) + throw new Error("container is detached"); + const pending = this.#ensurePending(); + const counter = this.#nextOperationCounter(pending); + const operation = { + container: container._codecId, + counter, + length, + content, + }; + if (preferredChild !== undefined) { + const childId = this.#childIdForOperation(operation, preferredChild.kind()); + preferredChild._attach(this, childId, container); + this.#containers.set(preferredChild.id, preferredChild); + } + if (!appendToTrailingListInsert(pending.operations, operation)) { + pending.operations.push(operation); + } + pending.operationLength += operation.length; + pending.changedContainers.add(container.id); + const causalVersion = pending.causalVersion; + causalVersion.set(pending.id.peer, Math.max(causalVersion.get(pending.id.peer) ?? 0, operation.counter)); + const finishEvent = this.#prepareEvent(pending, container, operation, pending.keys, pending.id, causalVersion); + this.#applyOperation(operation, pending.keys, pending.id, pending.lamport, causalVersion, container); + finishEvent?.(); + return operation; + } + #prepareEvent(recording, container, operation, keys, changeId, causalVersion, force = false) { + if (!force && !this.#hasEventSubscribers()) + return undefined; + const content = operation.content; + if (container instanceof LoroText) { + const state = this.#sequenceEventState(recording, container, "text"); + if (content.type === "text-insert") { + return () => { + const first = container._sequence.findById({ + peer: changeId.peer, + counter: operation.counter, + }); + if (first === undefined || first.deleted) + return; + const position = container._sequence.visibleMetricOffsetOf(first, "utf16"); + if (position === undefined) + return; + const attributes = Object.fromEntries([...container._attributesAt(first)].map(([key, value]) => [ + key, + runtimeValueToJson(value), + ])); + state.diff.insertText(position, content.value, attributes); + }; + } + if (content.type === "text-delete") { + for (const range of this.#sequenceEventDeletionRanges(container, content.startId, Math.abs(Number(content.length))).reverse()) { + state.diff.delete(range.position, range.length); + } + return undefined; + } + if (content.type === "text-mark") { + const value = this.#decodeRuntimeValue(content.value, keys, { peer: changeId.peer, counter: operation.counter }, container); + const runs = container._styleRuns(content.start, content.end, causalVersion); + for (const range of container._sequence.visibleMetricRangesForIdRuns(runs, "utf16")) { + state.diff.formatText(range.start, range.end - range.start, content.key, runtimeValueToJson(value)); + } + return undefined; + } + if (content.type === "text-mark-end") + return undefined; + } + else if (container instanceof LoroList) { + const state = this.#sequenceEventState(recording, container, "list"); + if (content.type === "list-insert" || content.type === "movable-list-insert") { + return () => { + const first = container._sequence.findById({ + peer: changeId.peer, + counter: operation.counter, + }); + if (first === undefined || first.deleted) + return; + const position = container._sequence.visibleIndexOf(first); + if (position === undefined) + return; + state.diff.insertList(position, container + ._visibleElementsRange(position, position + content.values.length) + .map((element) => cloneRuntimeValue(element.value))); + }; + } + if (content.type === "list-delete" || content.type === "movable-list-delete") { + for (const range of this.#sequenceEventDeletionRanges(container, content.startId, Math.abs(Number(content.length))).reverse()) { + state.diff.delete(range.position, range.length); + } + return undefined; + } + if (content.type === "movable-list-move") { + const element = container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + const from = element === undefined || element.deleted + ? undefined + : container._sequence.visibleIndexOf(element); + if (element === undefined || from === undefined) + return undefined; + const value = cloneRuntimeValue(element.value); + return () => { + const to = element.deleted + ? undefined + : container._sequence.visibleIndexOf(element); + if (to === undefined || to === from) + return; + state.diff.delete(from, 1); + state.diff.insertList(to, [value]); + }; + } + if (content.type === "movable-list-set") { + const element = container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + const position = element === undefined || element.deleted + ? undefined + : container._sequence.visibleIndexOf(element); + if (element === undefined || position === undefined) + return undefined; + const selected = element; + const previous = cloneRuntimeValue(element.value); + return () => { + const value = cloneRuntimeValue(selected.value); + if (eventValuesEqual(previous, value)) + return; + state.diff.delete(position, 1); + state.diff.insertList(position, [value]); + }; + } + } + else if (container instanceof LoroMap) { + if (content.type === "map-insert" || content.type === "map-delete") { + const state = this.#mapEventState(recording, container); + const beforeRecord = container._entries.get(content.key); + const beforeVisible = beforeRecord !== undefined && !beforeRecord.deleted; + const beforeValue = beforeVisible + ? cloneRuntimeValue(beforeRecord.value) + : undefined; + if (!state.originals.has(content.key)) { + state.originals.set(content.key, { + present: beforeRecord !== undefined, + visible: beforeVisible, + value: beforeValue, + }); + } + return () => { + const afterRecord = container._entries.get(content.key); + const afterVisible = afterRecord !== undefined && !afterRecord.deleted; + const afterValue = afterVisible + ? cloneRuntimeValue(afterRecord.value) + : undefined; + if (beforeVisible !== afterVisible || + !eventValuesEqual(beforeValue, afterValue)) { + state.changed.add(content.key); + } + }; + } + } + else if (container instanceof LoroTree) { + if (content.type === "tree-create" || + content.type === "tree-move" || + content.type === "tree-delete") { + const state = this.#treeEventState(recording, container); + const id = formatTreeId(content.subject); + if (!state.originals.has(id)) { + const record = container._nodes.get(id); + state.originals.set(id, record === undefined || container._isRecordDeleted(record) + ? undefined + : this.#treeEventNode(container, record)); + } + return undefined; + } + } + else if (container instanceof LoroCounter) { + if (!recording.eventStates.has(container.id)) { + recording.eventStates.set(container.id, { + kind: "counter", + before: container.value, + }); + } + return undefined; + } + if (!recording.beforeValues.has(container.id)) { + recording.beforeValues.set(container.id, containerEventValue(container)); + } + return undefined; + } + #sequenceEventState(recording, container, kind) { + const existing = recording.eventStates.get(container.id); + if (existing !== undefined) { + if (existing.kind !== "sequence") + throw new Error("event state kind changed"); + return existing; + } + const state = { + kind: "sequence", + diff: new SequenceEventDiff(kind, container.length), + }; + recording.eventStates.set(container.id, state); + return state; + } + #mapEventState(recording, container) { + const existing = recording.eventStates.get(container.id); + if (existing !== undefined) { + if (existing.kind !== "map") + throw new Error("event state kind changed"); + return existing; + } + const state = { + kind: "map", + originals: new Map(), + changed: new Set(), + }; + recording.eventStates.set(container.id, state); + return state; + } + #treeEventState(recording, container) { + const existing = recording.eventStates.get(container.id); + if (existing !== undefined) { + if (existing.kind !== "tree") + throw new Error("event state kind changed"); + return existing; + } + const state = { + kind: "tree", + originals: new Map(), + }; + recording.eventStates.set(container.id, state); + return state; + } + #treeEventNode(tree, record) { + return { + id: formatTreeId(record.id), + parent: record.parent === undefined ? undefined : formatTreeId(record.parent), + index: tree._indexOf(record), + fractionalIndex: bytesToHex(record.position).toUpperCase(), + }; + } + #sequenceEventDeletionRanges(container, startId, length) { + return container._sequence + .visibleMetricRangesForIdRuns([{ start: { peer: startId.peer, counter: startId.counter }, length }], "utf16") + .map(({ start, end }) => ({ position: start, length: end - start })); + } + #recordedEventDiffs(recording, includeMapTombstones = false) { + const diffs = new Map(); + for (const [id, state] of recording.eventStates) { + const container = this.#containers.get(id); + if (container === undefined) + continue; + if (state.kind === "sequence") { + diffs.set(id, state.diff.toDiff()); + } + else if (state.kind === "map" && container instanceof LoroMap) { + const updated = []; + for (const [key, original] of state.originals) { + const record = container._entries.get(key); + const present = record !== undefined; + const visible = record !== undefined && !record.deleted; + const value = visible ? cloneRuntimeValue(record.value) : undefined; + if (state.changed.has(key) || + (includeMapTombstones && present !== original.present) || + visible !== original.visible || + !eventValuesEqual(value, original.value)) { + updated.push([key, value]); + } + } + diffs.set(id, { type: "map", updated: Object.fromEntries(updated) }); + } + else if (state.kind === "counter" && container instanceof LoroCounter) { + diffs.set(id, { type: "counter", increment: container.value - state.before }); + } + else if (state.kind === "tree" && container instanceof LoroTree) { + const before = []; + const after = []; + for (const [nodeId, original] of state.originals) { + if (original !== undefined) + before.push(original); + const record = container._nodes.get(nodeId); + if (record !== undefined && !container._isRecordDeleted(record)) { + after.push(this.#treeEventNode(container, record)); + } + } + diffs.set(id, { type: "tree", diff: treeDelta(before, after) }); + } + } + return diffs; + } + #attachChild(parent, child, content) { + if (child._doc !== undefined && child._doc !== this) { + throw new Error("cannot attach a container from another document"); + } + const blueprint = captureBlueprint(child); + const attached = createContainer(child.kind()); + const rawType = containerTypeToRawByte(publicTypeToCodec(child.kind())); + this.#appendAndApply(parent, content(rawType), 1, attached); + restoreBlueprint(attached, blueprint); + if (!child.isAttached()) + child._attached = attached; + return attached; + } + #childIdForOperation(operation, type) { + return { + kind: "normal", + peer: this.#peer, + counter: operation.counter, + containerType: publicTypeToCodec(type), + }; + } + #deleteSequenceRuns(container, position, length, type) { + for (const run of container._sequence.visibleIdRuns(position, position + length)) { + this.#appendAndApply(container, { type, position, length: BigInt(run.length), startId: run.start }, run.length); + } + } + #deleteTextRuns(container, position, length) { + for (const run of container._sequence.visibleIdRuns(position, position + length)) { + this.#appendAndApply(container, { + type: "text-delete", + position, + length: BigInt(run.length), + startId: run.start, + }, run.length); + this.#mergeTrailingTextDeletes(); + } + } + #mergeTrailingTextDeletes() { + const operations = this.#pending?.operations; + if (operations === undefined || operations.length < 2) + return; + const left = operations[operations.length - 2]; + const right = operations[operations.length - 1]; + if (left.content.type !== "text-delete" || + right.content.type !== "text-delete" || + !containerIdsEqual(left.container, right.container)) { + return; + } + const leftLength = Number(left.content.length); + const rightLength = Number(right.content.length); + if (rightLength <= 0 || left.content.startId.peer !== right.content.startId.peer) { + return; + } + let mergedLength; + let startId = left.content.startId; + if (leftLength > 0 && + left.content.position === right.content.position && + right.content.startId.counter === left.content.startId.counter + leftLength) { + mergedLength = leftLength + rightLength; + } + else if (leftLength === 1 && + right.content.position + rightLength === left.content.position && + right.content.startId.counter + rightLength === left.content.startId.counter) { + mergedLength = -(leftLength + rightLength); + startId = right.content.startId; + } + else if (leftLength < 0 && + right.content.position + rightLength - 1 === left.content.position + leftLength && + right.content.startId.counter + rightLength === left.content.startId.counter) { + mergedLength = leftLength - rightLength; + startId = right.content.startId; + } + if (mergedLength === undefined) + return; + operations.splice(operations.length - 2, 2, { + ...left, + length: left.length + right.length, + content: { + ...left.content, + length: BigInt(mergedLength), + startId, + }, + }); + } + #encodeRuntimeValue(value, pending) { + if (value == null) + return { type: "null" }; + if (typeof value === "boolean") + return { type: "bool", value }; + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("Loro numbers must be finite"); + return Number.isSafeInteger(value) + ? { type: "i64", value: BigInt(value) } + : { type: "double", value }; + } + if (typeof value === "string") + return { type: "string", value }; + if (value instanceof Uint8Array) + return { type: "binary", value: value.slice() }; + if (Array.isArray(value)) { + return { + type: "list", + value: value.map((item) => this.#encodeRuntimeValue(item, pending)), + }; + } + if (isContainer(value)) + throw new TypeError("attach child containers with a container-specific method"); + if (typeof value === "object") { + if (Object.getOwnPropertySymbols(value).length > 0) { + throw new TypeError("Object keys must be strings"); + } + return { + type: "map", + value: Object.entries(value).map(([key, item]) => [ + BigInt(registerKey(pending, key)), + this.#encodeRuntimeValue(item, pending), + ]), + }; + } + throw new TypeError(`unsupported Loro value type: ${typeof value}`); + } + #decodeRuntimeValue(value, keys, operationId, parent) { + switch (value.type) { + case "null": + return null; + case "bool": + return value.value; + case "i64": + return Number(value.value); + case "double": + return value.value; + case "string": + return value.value; + case "binary": + return value.value.slice(); + case "list": + return value.value.map((item, index) => this.#decodeRuntimeValue(item, keys, { peer: operationId.peer, counter: operationId.counter + index }, parent)); + case "map": + return Object.fromEntries(value.value.map(([keyIndex, item]) => { + const key = keys[Number(keyIndex)]; + if (key === undefined) + throw new Error("change value map key index is out of range"); + return [key, this.#decodeRuntimeValue(item, keys, operationId, parent)]; + })); + case "container-type": { + const containerType = containerTypeFromRawByte(value.value); + const id = { kind: "normal", ...operationId, containerType }; + return this.#getOrCreateContainer(id, parent); + } + } + } + #applyOperation(operation, keys, changeId, changeLamport, causalVersion, knownContainer) { + const container = knownContainer ?? this.#getOrCreateContainer(operation.container); + const operationId = { peer: changeId.peer, counter: operation.counter }; + const lamport = changeLamport + (operation.counter - changeId.counter); + const content = operation.content; + switch (content.type) { + case "map-insert": + { + const rawValue = this.#decodeRuntimeValue(content.value, keys, operationId, container); + const writer = { peer: changeId.peer, lamport }; + container._applyValue(content.key, this.#materializeMapValue(container, content.key, rawValue), writer, rawValue); + } + return; + case "map-delete": + container._applyDelete(content.key, { + peer: changeId.peer, + lamport, + }); + return; + case "list-insert": + case "movable-list-insert": { + const values = content.values.map((value, index) => this.#decodeRuntimeValue(value, keys, { peer: operationId.peer, counter: operationId.counter + index }, container)); + container._insertFugue(content.position, values, values.map((_, index) => ({ + peer: operationId.peer, + counter: operationId.counter + index, + })), values.map((_, index) => lamport + index), causalVersion); + return; + } + case "list-delete": + case "movable-list-delete": + container._deleteIdSpan(content.startId, Number(content.length), operationId); + return; + case "text-insert": { + container._insertFugue(content.position, content.value, operationId, lamport, causalVersion); + return; + } + case "text-delete": + container._deleteIdSpan(content.startId, Number(content.length), operationId); + return; + case "text-mark": + { + const value = this.#decodeRuntimeValue(content.value, keys, operationId, container); + const meta = { + startId: operationId, + lamport, + info: content.info, + value, + }; + container._applyMark(content.start, content.end, content.key, value, meta, causalVersion); + } + return; + case "text-mark-end": + return; + case "movable-list-move": { + const list = container; + const element = list._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + const from = element === undefined || element.deleted + ? undefined + : list._sequence.visibleIndexOf(element); + if (from !== undefined) + list._applyMove(from, Math.min(content.to, list.length - 1), { + id: operationId, + lamport, + }); + return; + } + case "movable-list-set": { + const list = container; + const element = list._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + if (element !== undefined) { + const value = this.#decodeRuntimeValue(content.value, keys, operationId, container); + list._applySet(element, value, { + id: operationId, + lamport, + value, + }); + } + return; + } + case "tree-create": + case "tree-move": { + const tree = container; + const writer = { peer: changeId.peer, lamport }; + const nodeKey = formatTreeId(content.subject); + let record = tree._nodes.get(nodeKey); + if (record === undefined) { + const dataId = { + kind: "normal", + ...content.subject, + containerType: CodecContainerType.Map, + }; + const data = this.#getOrCreateContainer(dataId, tree); + record = { + id: content.subject, + parent: content.parent, + position: content.position.slice(), + deleted: false, + writer, + lastMoveId: operationId, + data, + }; + tree._setRecord(record); + } + else if (compareWriter(record.writer, writer) <= 0) { + tree._updateRecord(record, content.parent, content.position.slice(), writer, operationId); + } + return; + } + case "tree-delete": { + const record = container._nodes.get(formatTreeId(content.subject)); + if (record !== undefined) { + const writer = { peer: changeId.peer, lamport }; + if (compareWriter(record.writer, writer) <= 0) { + container._deleteRecord(record, writer, operationId); + } + } + return; + } + case "future": + if (container instanceof LoroCounter) { + if (content.value.type === "double" || content.value.type === "i64") { + container._value += + content.value.type === "double" + ? content.value.value + : Number(content.value.value); + } + else if (content.value.type === "delta-int") { + container._value += content.value.value; + } + } + } + } + #readChangeBlock(bytes) { + const block = decodeChangeBlock(bytes); + return block.changes.map((change) => ({ + change, + keys: block.keys, + keyIndices: undefined, + })); + } + #decodeImportData(parsed) { + if (parsed.mode === EncodeMode.FastUpdates) { + return { + mode: parsed.mode, + records: decodeFastUpdatesBody(parsed.body).flatMap((block) => this.#readChangeBlock(block)), + }; + } + const snapshot = decodeFastSnapshotBody(parsed.body); + const records = []; + let startVersion; + let startFrontiers; + let endVersion; + for (const entry of decodeSstable(snapshot.oplog)) { + if (bytesEqual(entry.key, VERSION_KEY)) { + endVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); + } + else if (bytesEqual(entry.key, START_VERSION_KEY)) { + startVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); + } + else if (bytesEqual(entry.key, START_FRONTIERS_KEY)) { + startFrontiers = decodePostcardFrontiers(entry.value); + } + else if (entry.key.length === 12) { + const expected = decodeChangeBlockKey(entry.key); + const block = this.#readChangeBlock(entry.value); + if (block[0] !== undefined && !idsEqual(block[0].change.id, expected)) { + throw new Error("snapshot change key does not match its block"); + } + records.push(...block); + } + } + return { + mode: parsed.mode, + records, + snapshot, + ...(startVersion === undefined ? {} : { startVersion }), + ...(startFrontiers === undefined ? {} : { startFrontiers }), + ...(endVersion === undefined ? {} : { endVersion }), + }; + } + #integrateHistory(records, mergeInterval = 0n) { + const added = []; + const addedRecordIndices = new Map(); + for (const record of records) { + const key = changeKey(record.change.id); + const knownEnd = Math.max(this.#shallowStartVersion.get(record.change.id.peer) ?? 0, this.#historyEndByPeer.get(record.change.id.peer) ?? 0); + const recordEnd = record.change.id.counter + changeLength(record.change); + if (recordEnd <= knownEnd) + continue; + const pending = this.#pendingHistory.get(key); + if (pending !== undefined && + pending.change.id.counter + changeLength(pending.change) >= recordEnd) + continue; + this.#pendingHistory.set(key, record); + } + // Hoist the sorted pending queue and the causal version out of the + // promotion loop; the version is advanced locally as records promote and + // stays in sync with #historyEndByPeer. The queue is compacted in place + // after each pass, and only rebuilt from the map when a sliced record was + // re-keyed (which may have overwritten another pending entry). + const version = this.#historyVersion(); + let queue = [...this.#pendingHistory.values()].sort(compareHistoryRecords); + let promoted = true; + while (promoted) { + promoted = false; + let rebuildQueue = false; + let kept = 0; + for (let index = 0; index < queue.length; index += 1) { + let record = queue[index]; + let change = record.change; + let key = changeKey(change.id); + const knownEnd = version.get(change.id.peer) ?? 0; + const changeEnd = change.id.counter + changeLength(change); + if (change.id.counter < knownEnd) { + this.#pendingHistory.delete(key); + if (changeEnd <= knownEnd) { + promoted = true; + continue; + } + record = { + ...record, + change: sliceChange(change, knownEnd - change.id.counter, changeLength(change)), + }; + change = record.change; + key = changeKey(change.id); + this.#pendingHistory.set(key, record); + promoted = true; + rebuildQueue = true; + } + if (change.id.counter !== (version.get(change.id.peer) ?? 0)) { + queue[kept] = record; + kept += 1; + continue; + } + if (change.dependencies.some((dependency) => !this.#historyContainsId(dependency))) { + queue[kept] = record; + kept += 1; + continue; + } + this.#pendingHistory.delete(key); + const previous = mergeInterval === undefined + ? undefined + : this.#mergeablePreviousRecord(change, mergeInterval); + if (previous === undefined) { + this.#setHistoryRecord(key, record, record); + addedRecordIndices.set(record, added.length); + } + else { + const addedPreviousIndex = addedRecordIndices.get(previous); + if (addedPreviousIndex !== undefined) { + added[addedPreviousIndex] = cloneHistoryRecord(previous); + addedRecordIndices.delete(previous); + } + const previousLength = changeLength(previous.change); + const appended = appendHistoryRecord(previous, record, previousLength); + this.#appendMergedHistoryRecord(previous, appended, previousLength); + } + version.set(change.id.peer, change.id.counter + changeLength(change)); + added.push(record); + this.#seenCommittedPeers.add(change.id.peer); + promoted = true; + } + queue.length = kept; + if (rebuildQueue) { + queue = [...this.#pendingHistory.values()].sort(compareHistoryRecords); + } + } + return { + added: added.sort(compareHistoryRecords), + pending: [...this.#pendingHistory.values()].sort(compareHistoryRecords), + }; + } + #historyContainsId(id) { + if (id.counter < (this.#shallowStartVersion.get(id.peer) ?? 0)) + return true; + return id.counter < (this.#historyEndByPeer.get(id.peer) ?? 0); + } + #assertImportsNotOutdated(records) { + if (this.#shallowRootStore === undefined) + return; + for (const { change } of records) { + const knownEnd = this.#shallowStartVersion.get(change.id.peer) ?? 0; + if (change.id.counter + changeLength(change) <= knownEnd) + continue; + if (change.dependencies.length === 0) { + throw new Error("cannot import updates that depend on an outdated version"); + } + const touchesShallowRoot = change.dependencies.some((dependency) => this.#shallowRootFrontiers.some((frontier) => idsEqual(frontier, dependency))); + if (touchesShallowRoot) + continue; + const dependsOnPrunedHistory = change.dependencies.some((dependency) => dependency.counter < (this.#shallowStartVersion.get(dependency.peer) ?? 0)); + if (dependsOnPrunedHistory) { + throw new Error("cannot import updates that depend on an outdated version"); + } + } + } + #applyRecords(records, recording) { + for (const record of records) { + const causalVersion = this.#causalVersionAt(record.change.dependencies); + for (const operation of record.change.operations) { + const container = this.#getOrCreateContainer(operation.container); + const finishEvent = recording === undefined + ? undefined + : this.#prepareEvent(recording, container, operation, record.keys, record.change.id, causalVersion, true); + this.#applyOperation(operation, record.keys, record.change.id, record.change.lamport, causalVersion, container); + finishEvent?.(); + causalVersion.set(record.change.id.peer, Math.max(causalVersion.get(record.change.id.peer) ?? 0, operation.counter + operation.length)); + } + } + } + #canTransitionRecords(records, movableMoveMode = "anchors") { + const replayedMoveContainers = new Set(); + if (movableMoveMode === "replay") { + for (const { change } of records) { + for (const operation of change.operations) { + if (operation.content.type === "movable-list-move") { + const containerId = this.#containerKey(operation.container); + replayedMoveContainers.add(containerId); + } + } + } + for (const containerId of replayedMoveContainers) { + if (this.#movableOrderHistory.get(containerId) === undefined) + return false; + } + } + const moveSuffixes = new Map(); + for (const { change } of records) { + const causalVersion = this.#causalVersionAt(change.dependencies); + for (const operation of change.operations) { + const container = this.#containers.get(this.#containerKey(operation.container)); + const content = operation.content; + if (content.type === "movable-list-move") { + const element = container instanceof LoroMovableList + ? container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport) + : undefined; + const history = element?.moveHistory; + const moveIndex = findSequenceMoveMetaIndex(history, operationWriter(change, operation)); + if (!(container instanceof LoroMovableList) || + !container._moveHistoryComplete || + element === undefined || + (movableMoveMode === "anchors" && + (history === undefined || + moveIndex < 0 || + history[moveIndex].id.peer !== change.id.peer || + history[moveIndex].id.counter !== operation.counter))) { + return false; + } + if (movableMoveMode === "anchors") { + let suffix = moveSuffixes.get(element); + if (suffix === undefined) { + suffix = { history: history, indices: new Set() }; + moveSuffixes.set(element, suffix); + } + suffix.indices.add(moveIndex); + } + } + else if (content.type === "movable-list-set") { + const element = container instanceof LoroMovableList + ? container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport) + : undefined; + if (!(container instanceof LoroMovableList) || + !container._valueHistoryComplete || + element === undefined || + !hasSequenceValueMeta(element.valueHistory, operationWriter(change, operation), change.id.peer, operation.counter)) { + return false; + } + } + else if (content.type === "text-mark") { + if (!(container instanceof LoroText) || !container._attributeHistoryComplete) { + return false; + } + const viewLength = container._sequence.isFullyIncluded(causalVersion) + ? container._sequence.visibleLength + : container._sequence.causalView(causalVersion).length; + if (content.end > viewLength) { + return false; + } + const runs = container._styleRuns(content.start, content.end, causalVersion); + if (!container._styleIndex.runsContainMeta(runs, content.key, { + peer: change.id.peer, + counter: operation.counter, + })) { + return false; + } + } + else if (content.type === "text-mark-end") { + if (!(container instanceof LoroText)) + return false; + } + else if (content.type === "text-insert") { + if (!(container instanceof LoroText)) + return false; + scratchSequenceIdRun.start.peer = change.id.peer; + scratchSequenceIdRun.start.counter = operation.counter; + scratchSequenceIdRun.length = operation.length; + if (!container._sequence.containsIdRuns(scratchSequenceIdRuns)) { + return false; + } + } + else if (content.type === "list-insert" || + content.type === "movable-list-insert") { + if (!(container instanceof LoroList)) + return false; + scratchSequenceIdRun.start.peer = change.id.peer; + scratchSequenceIdRun.start.counter = operation.counter; + scratchSequenceIdRun.length = operation.length; + if (!container._sequence.containsIdRuns(scratchSequenceIdRuns)) { + return false; + } + } + else if (content.type === "text-delete" || + content.type === "list-delete" || + content.type === "movable-list-delete") { + if (!(container instanceof LoroList || container instanceof LoroText)) { + return false; + } + } + else if (content.type === "map-insert" || content.type === "map-delete") { + if (!(container instanceof LoroMap)) + return false; + } + else if (content.type === "tree-create" || + content.type === "tree-move" || + content.type === "tree-delete") { + if (!(container instanceof LoroTree)) + return false; + } + else if (content.type === "future") { + if (!(container instanceof LoroCounter) || + counterDelta(content) === undefined) { + return false; + } + } + causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); + } + } + for (const { history, indices } of moveSuffixes.values()) { + const first = Math.min(...indices); + if (history.length - first !== indices.size) + return false; + for (let index = first; index < history.length; index += 1) { + if (!indices.has(index)) + return false; + } + } + return true; + } + #applyVersionTransition(retreat, forward, target, recording, movableMoveMode = "anchors") { + const sequences = new Map(); + const bulkSequenceRemovals = new Map(); + const bulkSequenceRestorations = new Map(); + const textAttributeRuns = new Map(); + const textStyleContainers = new Set(); + const movableValues = new Map(); + const mapKeys = new Map(); + const treeSubjects = new Map(); + const counters = new Map(); + const sequenceElements = (container) => { + let elements = sequences.get(container); + if (elements === undefined) { + elements = new Set(); + sequences.set(container, elements); + } + return elements; + }; + const addTextAttributeRuns = (container, key, runs) => { + let keys = textAttributeRuns.get(container); + if (keys === undefined) { + keys = new Map(); + textAttributeRuns.set(container, keys); + } + const existing = keys.get(key); + if (existing === undefined) + keys.set(key, [...runs]); + else + existing.push(...runs); + }; + const addBulkSequenceRemovals = (container, runs) => { + const existing = bulkSequenceRemovals.get(container); + if (existing === undefined) + bulkSequenceRemovals.set(container, [...runs]); + else + existing.push(...runs); + }; + const addBulkSequenceRestorations = (container, runs) => { + const existing = bulkSequenceRestorations.get(container); + if (existing === undefined) + bulkSequenceRestorations.set(container, [...runs]); + else + existing.push(...runs); + }; + const collect = (records, direction) => { + for (const { change } of records) { + const causalVersion = this.#causalVersionAt(change.dependencies); + for (const operation of change.operations) { + const container = this.#containers.get(this.#containerKey(operation.container)); + const content = operation.content; + if (content.type === "text-insert" || + content.type === "list-insert" || + content.type === "movable-list-insert") { + const sequenceContainer = container; + const sequence = sequenceContainer._sequence; + scratchSequenceIdRun.start.peer = change.id.peer; + scratchSequenceIdRun.start.counter = operation.counter; + scratchSequenceIdRun.length = operation.length; + if (direction === -1 || + (recording === undefined && + sequence.canShowIdRunsAt(scratchSequenceIdRuns, target))) { + // The bulk lists retain their runs, so they get a fresh copy + // rather than the shared scratch run. + const insertedRuns = [ + { + start: { peer: change.id.peer, counter: operation.counter }, + length: operation.length, + }, + ]; + if (direction === -1) { + addBulkSequenceRemovals(sequenceContainer, insertedRuns); + } + else { + addBulkSequenceRestorations(sequenceContainer, insertedRuns); + } + causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); + continue; + } + const elements = sequenceElements(sequenceContainer); + for (let offset = 0; offset < operation.length; offset += 1) { + elements.add(sequence.findById({ + peer: change.id.peer, + counter: operation.counter + offset, + })); + } + } + else if (content.type === "text-delete" || + content.type === "list-delete" || + content.type === "movable-list-delete") { + const sequenceContainer = container; + const deletedRuns = sequenceContainer._sequence.idRunsDeletedBy(change.id.peer, operation.counter, operation.counter + operation.length); + if (direction === 1 || + (recording === undefined && + sequenceContainer._sequence.canShowIdRunsAt(deletedRuns, target))) { + if (direction === 1) { + addBulkSequenceRemovals(sequenceContainer, deletedRuns); + } + else { + addBulkSequenceRestorations(sequenceContainer, deletedRuns); + } + causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); + continue; + } + const elements = sequenceElements(sequenceContainer); + for (const element of sequenceContainer._sequence.elementsDeletedBy(change.id.peer, operation.counter, operation.counter + operation.length)) { + elements.add(element); + } + } + else if (content.type === "text-mark") { + const text = container; + textStyleContainers.add(text); + if (recording !== undefined) { + addTextAttributeRuns(text, content.key, text._styleRuns(content.start, content.end, causalVersion)); + } + } + else if (content.type === "movable-list-set") { + const list = container; + const element = list._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + let elements = movableValues.get(list); + if (elements === undefined) { + elements = new Set(); + movableValues.set(list, elements); + } + elements.add(element); + } + else if (content.type === "map-insert" || content.type === "map-delete") { + const map = container; + let keys = mapKeys.get(map); + if (keys === undefined) { + keys = new Set(); + mapKeys.set(map, keys); + } + keys.add(content.key); + } + else if (content.type === "tree-create" || + content.type === "tree-move" || + content.type === "tree-delete") { + const tree = container; + let subjects = treeSubjects.get(tree); + if (subjects === undefined) { + subjects = new Map(); + treeSubjects.set(tree, subjects); + } + subjects.set(idKey(content.subject), content.subject); + } + else if (content.type === "future") { + const counter = container; + counters.set(counter, (counters.get(counter) ?? 0) + direction * counterDelta(content)); + } + causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); + } + } + }; + collect(retreat, -1); + collect(forward, 1); + if (recording !== undefined) { + for (const [container, runs] of [...bulkSequenceRemovals]) { + if (!sequences.has(container)) + continue; + const elements = sequenceElements(container); + for (const run of runs) { + for (let offset = 0; offset < run.length; offset += 1) { + const element = container._sequence.findById({ + peer: run.start.peer, + counter: run.start.counter + offset, + }); + if (element !== undefined) + elements.add(element); + } + } + bulkSequenceRemovals.delete(container); + } + } + const includes = (id) => id.counter < (target.get(id.peer) ?? 0); + const targetDeleted = (container, element) => !includes(element.id) || container._sequence.someDeletion(element, includes); + const styleVersion = target.compare(this.#historyVersion()) === 0 + ? undefined + : new Map(target + ._codecEntriesUnsorted() + .map(({ peer, counter }) => [peer, counter])); + for (const [text, keys] of textAttributeRuns) { + const state = this.#sequenceEventState(recording, text, "text"); + const removedRuns = [ + ...(bulkSequenceRemovals.get(text) ?? []), + ...[...(sequences.get(text) ?? [])] + .filter((element) => !element.deleted && targetDeleted(text, element)) + .map((element) => ({ start: element.id, length: 1 })), + ]; + for (const [key, runs] of keys) { + for (const transition of text._styleIndex.transitions(runs, key, text._styleVersion, styleVersion)) { + const beforePresent = transition.before !== undefined && transition.before.value !== null; + const beforeValue = beforePresent ? transition.before.value : undefined; + const afterPresent = transition.after !== undefined && transition.after.value !== null; + const afterValue = afterPresent ? transition.after.value : undefined; + if (beforePresent === afterPresent && + eventValuesEqual(beforeValue, afterValue)) { + continue; + } + const retainedRuns = subtractSequenceIdRuns([transition.run], removedRuns); + for (const range of text._sequence.visibleMetricRangesForIdRuns(retainedRuns, "utf16")) { + state.diff.formatText(range.start, range.end - range.start, key, afterPresent ? runtimeValueToJson(afterValue) : null); + } + } + } + } + for (const text of textStyleContainers) + text._setStyleVersion(styleVersion); + for (const [container, runs] of bulkSequenceRestorations) { + container._sequence.setIdRunsVisible(runs); + } + for (const [container, elements] of sequences) { + const state = recording === undefined + ? undefined + : this.#sequenceEventState(recording, container, container instanceof LoroText ? "text" : "list"); + const removals = [...elements] + .filter((element) => !element.deleted && targetDeleted(container, element)) + .map((element) => ({ + element, + position: container instanceof LoroText + ? container._sequence.visibleMetricOffsetOf(element, "utf16") + : container._sequence.visibleIndexOf(element), + })) + .sort((left, right) => right.position - left.position); + for (const { element, position } of removals) { + state?.diff.delete(position, container instanceof LoroText ? element.value.length : 1); + container._sequence.setDeleted(element, true); + } + const insertions = [...elements] + .filter((element) => element.deleted && !targetDeleted(container, element)) + .sort((left, right) => container._sequence.physicalIndexOf(left) - + container._sequence.physicalIndexOf(right)); + for (const element of insertions) { + container._sequence.setDeleted(element, false); + if (state === undefined) + continue; + if (container instanceof LoroText) { + const textElement = element; + const position = container._sequence.visibleMetricOffsetOf(textElement, "utf16"); + const attributes = Object.fromEntries([...container._attributesAt(textElement)].map(([key, value]) => [ + key, + runtimeValueToJson(value), + ])); + state.diff.insertText(position, textElement.value, attributes); + } + else { + const position = container._sequence.visibleIndexOf(element); + state.diff.insertList(position, [cloneRuntimeValue(element.value)]); + } + } + } + for (const [container, runs] of bulkSequenceRemovals) { + if (recording !== undefined) { + const state = this.#sequenceEventState(recording, container, container instanceof LoroText ? "text" : "list"); + for (const range of container._sequence + .visibleMetricRangesForIdRuns(runs, "utf16") + .reverse()) { + state.diff.delete(range.start, range.end - range.start); + } + } + container._sequence.setIdRunsDeleted(runs); + } + if (movableMoveMode === "replay") { + this.#replayMovableMoves(retreat, forward, target, recording); + } + else { + this.#transitionMovableMoves(retreat, true, recording); + this.#transitionMovableMoves(forward, false, recording); + } + for (const [list, elements] of movableValues) { + const state = recording === undefined + ? undefined + : this.#sequenceEventState(recording, list, "list"); + for (const element of elements) { + const winner = latestIncludedSequenceValue(element.valueHistory, target); + if (winner === undefined || eventValuesEqual(element.value, winner.value)) { + continue; + } + if (!element.deleted) { + const position = list._sequence.visibleIndexOf(element); + if (position !== undefined) { + state?.diff.delete(position, 1); + state?.diff.insertList(position, [cloneRuntimeValue(winner.value)]); + } + } + element.value = winner.value; + list._bindChildren([element]); + } + } + for (const [map, keys] of mapKeys) { + const history = this.#mapOperationHistory.get(map.id); + for (const key of keys) { + if (recording !== undefined) { + const state = this.#mapEventState(recording, map); + if (!state.originals.has(key)) { + const record = map._entries.get(key); + state.originals.set(key, { + present: record !== undefined, + visible: record !== undefined && !record.deleted, + value: record === undefined || record.deleted + ? undefined + : cloneRuntimeValue(record.value), + }); + } + } + const indexed = latestIncludedOperation(history?.get(key), target); + if (indexed === undefined) { + map._replaceRecord(key, undefined); + continue; + } + const content = indexed.operation.content; + if (content.type === "map-delete") { + map._replaceRecord(key, { + value: undefined, + rawValue: undefined, + deleted: true, + writer: indexed.writer, + }); + } + else if (content.type === "map-insert") { + const operationId = { + peer: indexed.record.change.id.peer, + counter: indexed.operation.counter, + }; + const rawValue = this.#decodeRuntimeValue(content.value, indexed.record.keys, operationId, map); + map._replaceRecord(key, { + value: this.#materializeMapValue(map, key, rawValue), + rawValue, + deleted: false, + writer: indexed.writer, + }); + } + } + } + if (recording !== undefined) { + for (const [tree, subjects] of treeSubjects) { + const state = this.#treeEventState(recording, tree); + for (const subject of subjects.values()) { + const nodeKey = formatTreeId(subject); + if (state.originals.has(nodeKey)) + continue; + const record = tree._nodes.get(nodeKey); + state.originals.set(nodeKey, record === undefined || tree._isRecordDeleted(record) + ? undefined + : this.#treeEventNode(tree, record)); + } + } + } + for (const [tree, subjects] of treeSubjects) { + const history = this.#treeOperationHistory.get(tree.id); + for (const [historyKey, subject] of subjects) { + const nodeKey = formatTreeId(subject); + const operations = history?.get(historyKey); + const winner = latestIncludedOperation(operations, target); + const existing = tree._nodes.get(nodeKey); + if (winner === undefined) { + if (existing !== undefined) + tree._removeRecord(existing); + continue; + } + const winnerContent = winner.operation.content; + const placement = winnerContent.type === "tree-create" || winnerContent.type === "tree-move" + ? winner + : latestIncludedTreePlacement(operations, target, winner.writer); + if (placement === undefined) + continue; + const placementContent = placement.operation.content; + if (placementContent.type !== "tree-create" && + placementContent.type !== "tree-move") { + continue; + } + let record = tree._nodes.get(nodeKey); + const lastMoveId = { + peer: placement.record.change.id.peer, + counter: placement.operation.counter, + }; + if (record === undefined) { + const data = this.#getOrCreateContainer({ + kind: "normal", + ...placementContent.subject, + containerType: CodecContainerType.Map, + }, tree); + record = { + id: placementContent.subject, + parent: placementContent.parent, + position: placementContent.position.slice(), + deleted: false, + writer: placement.writer, + lastMoveId, + data, + }; + tree._setRecord(record); + } + else { + tree._updateRecord(record, placementContent.parent, placementContent.position.slice(), placement.writer, lastMoveId); + } + if (winnerContent.type === "tree-delete") { + tree._deleteRecord(record, winner.writer, { + peer: winner.record.change.id.peer, + counter: winner.operation.counter, + }); + } + } + } + for (const [counter, adjustment] of counters) { + if (recording !== undefined && !recording.eventStates.has(counter.id)) { + recording.eventStates.set(counter.id, { + kind: "counter", + before: counter.value, + }); + } + counter._value += adjustment; + } + } + #transitionMovableMoves(records, retreat, recording) { + const operations = records + .flatMap(({ change }) => change.operations.flatMap((operation) => operation.content.type === "movable-list-move" ? [{ change, operation }] : [])) + .sort((left, right) => compareHistoryOperations(left.change, left.operation, right.change, right.operation)); + if (retreat) + operations.reverse(); + for (const { change, operation } of operations) { + const content = operation.content; + if (content.type !== "movable-list-move") + continue; + const container = this.#containers.get(this.#containerKey(operation.container)); + if (!(container instanceof LoroMovableList)) + continue; + const element = container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + if (element === undefined || element.deleted) + continue; + const meta = findSequenceMoveMeta(element.moveHistory, operationWriter(change, operation)); + if (meta === undefined) + continue; + const from = container._sequence.visibleIndexOf(element); + if (from === undefined) + continue; + const value = cloneRuntimeValue(element.value); + container._moveToAnchors(element, retreat ? meta.beforePrevious : meta.afterPrevious, retreat ? meta.beforeNext : meta.afterNext); + const to = container._sequence.visibleIndexOf(element); + if (recording === undefined || to === undefined || to === from) + continue; + const state = this.#sequenceEventState(recording, container, "list"); + state.diff.delete(from, 1); + state.diff.insertList(to, [value]); + } + } + #canonicalizeImportedMovableMoves(records, recording) { + let hasMove = false; + for (const { change } of records) { + for (const operation of change.operations) { + if (operation.content.type !== "movable-list-move") + continue; + hasMove = true; + const container = this.#containers.get(this.#containerKey(operation.container)); + if (!(container instanceof LoroMovableList) || !container._moveHistoryComplete) { + return; + } + } + } + if (hasMove) + this.#replayMovableMoves([], records, this.#historyVersion(), recording); + } + #replayMovableMoves(retreat, forward, target, recording) { + const containerIds = new Set(); + for (const { change } of [...retreat, ...forward]) { + for (const operation of change.operations) { + if (operation.content.type === "movable-list-move") { + containerIds.add(this.#containerKey(operation.container)); + } + } + } + for (const containerId of containerIds) { + const container = this.#containers.get(containerId); + const history = this.#movableOrderHistory.get(containerId); + if (!(container instanceof LoroMovableList) || history === undefined) + continue; + const replay = new LoroMovableList(); + const ordered = history + .values() + .map((indexed) => ({ + indexed, + orderRecord: this.#recordContaining({ + peer: indexed.record.change.id.peer, + counter: indexed.operation.counter, + }) ?? indexed.record, + })) + .sort((left, right) => compareHistoryRecords(left.orderRecord, right.orderRecord) || + left.indexed.operation.counter - right.indexed.operation.counter); + for (const { indexed } of ordered) { + const peer = indexed.record.change.id.peer; + const includedLength = Math.min(indexed.operation.length, (target.get(peer) ?? 0) - indexed.operation.counter); + if (includedLength <= 0) + continue; + const operation = includedLength === indexed.operation.length + ? indexed.operation + : sliceOperation(indexed.operation, 0, includedLength); + const content = operation.content; + const operationId = { peer, counter: operation.counter }; + const lamport = indexed.record.change.lamport + + operation.counter - + indexed.record.change.id.counter; + const causalVersion = this.#causalVersionAt(indexed.record.change.dependencies); + causalVersion.set(peer, Math.max(causalVersion.get(peer) ?? 0, operation.counter)); + if (content.type === "movable-list-insert") { + replay._insertFugue(content.position, content.values.map(() => null), content.values.map((_, offset) => ({ + peer, + counter: operation.counter + offset, + })), content.values.map((_, offset) => lamport + offset), causalVersion); + } + else if (content.type === "movable-list-delete") { + replay._deleteIdSpan(content.startId, Number(content.length), operationId); + } + else if (content.type === "movable-list-move") { + const element = replay._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); + const from = element === undefined || element.deleted + ? undefined + : replay._sequence.visibleIndexOf(element); + if (from !== undefined) { + replay._applyMove(from, Math.min(content.to, replay.length - 1), { + id: operationId, + lamport, + }); + } + } + } + const current = container._visibleElements(); + const currentIndex = new Map(current.map((element, index) => [element, index])); + const targetElements = replay._visibleElements().map((replayed) => { + const element = container._sequence.findById(replayed.id); + if (element === undefined || element.deleted) { + throw new Error("movable-list replay produced an unavailable target element"); + } + return element; + }); + for (const replayed of replay._elements) { + const element = container._sequence.findById(replayed.id); + if (element === undefined) + continue; + for (const meta of replayed.moveHistory ?? []) { + let history = element.moveHistory; + if (history === undefined) { + history = []; + element.moveHistory = history; + } + const existing = history.findIndex(({ id }) => id.peer === meta.id.peer && id.counter === meta.id.counter); + if (existing >= 0) + history[existing] = meta; + else { + history.push(meta); + history.sort((left, right) => compareWriter({ peer: left.id.peer, lamport: left.lamport }, { peer: right.id.peer, lamport: right.lamport })); + } + } + } + if (targetElements.length !== current.length || + targetElements.some((element) => !currentIndex.has(element))) { + throw new Error("movable-list replay changed the visible element set"); + } + const stableTargetIndices = longestIncreasingSubsequenceIndices(targetElements.map((element) => currentIndex.get(element))); + const state = recording === undefined + ? undefined + : this.#sequenceEventState(recording, container, "list"); + for (let index = targetElements.length - 1; index >= 0; index -= 1) { + if (stableTargetIndices.has(index)) + continue; + const element = targetElements[index]; + const from = container._sequence.visibleIndexOf(element); + container._sequence.moveBefore(element, targetElements[index + 1]); + const to = container._sequence.visibleIndexOf(element); + if (state !== undefined && from !== to) { + state.diff.delete(from, 1); + state.diff.insertList(to, [cloneRuntimeValue(element.value)]); + } + } + } + } + #rebuildFromHistory(version) { + for (const container of this.#containers.values()) + container._reset(); + if (this.#shallowRootStore !== undefined) { + this.#hydrateState(this.#shallowRootStore); + const target = version ?? this.#historyVersion(); + this.#assertVersionNotBeforeShallowRoot(target); + this.#applyRecords(this.#recordsInVersionRange(this.#shallowRootVersion, target)); + return; + } + this.#applyRecords(version === undefined ? this.#sortedHistory() : this.#recordsAtVersion(version)); + } + #setHistoryRecord(key, record, appended) { + const previous = this.#history.get(key); + this.#history.set(key, record); + if (previous !== undefined) + this.#historyOrder.delete(previous); + this.#historyOrder.add(record); + this.#historyOperationCount += + changeLength(record.change) - + (previous === undefined ? 0 : changeLength(previous.change)); + this.#sortedHistoryCache = undefined; + let peerRecords = this.#historyByPeer.get(record.change.id.peer); + if (peerRecords === undefined) { + peerRecords = []; + this.#historyByPeer.set(record.change.id.peer, peerRecords); + } + if (previous !== undefined) { + const previousIndex = lowerBoundHistory(peerRecords, previous.change.id.counter); + if (peerRecords[previousIndex] === previous) + peerRecords.splice(previousIndex, 1); + } + const index = lowerBoundHistory(peerRecords, record.change.id.counter); + peerRecords.splice(index, 0, record); + const last = peerRecords.at(-1); + if (last === undefined) { + this.#historyEndByPeer.delete(record.change.id.peer); + } + else { + this.#historyEndByPeer.set(record.change.id.peer, last.change.id.counter + changeLength(last.change)); + } + if (previous !== undefined) { + const previousLast = { + peer: previous.change.id.peer, + counter: previous.change.id.counter + changeLength(previous.change) - 1, + }; + this.#historyFrontiers.delete(idKey(previousLast)); + } + for (const dependency of appended.change.dependencies) { + this.#historyFrontiers.delete(idKey(dependency)); + } + const lastId = { + peer: record.change.id.peer, + counter: record.change.id.counter + changeLength(record.change) - 1, + }; + this.#historyFrontiers.set(idKey(lastId), lastId); + this.#dependencyVersion(record.change); + this.#indexHistoryOperations(appended); + } + #appendMergedHistoryRecord(record, appended, previousLength) { + const appendedLength = changeLength(appended.change); + this.#historyOperationCount += appendedLength; + this.#historyEndByPeer.set(record.change.id.peer, record.change.id.counter + previousLength + appendedLength); + const previousLast = { + peer: record.change.id.peer, + counter: record.change.id.counter + previousLength - 1, + }; + this.#historyFrontiers.delete(idKey(previousLast)); + for (const dependency of appended.change.dependencies) { + this.#historyFrontiers.delete(idKey(dependency)); + } + const lastId = { + peer: record.change.id.peer, + counter: record.change.id.counter + previousLength + appendedLength - 1, + }; + this.#historyFrontiers.set(idKey(lastId), lastId); + this.#indexHistoryOperations(appended); + } + #indexHistoryOperations(record) { + for (const operation of record.change.operations) { + this.#containersWithOperations.add(this.#containerKey(operation.container)); + const content = operation.content; + if (content.type === "movable-list-insert" || + content.type === "movable-list-delete" || + content.type === "movable-list-move") { + const indexed = { + record, + operation, + writer: operationWriter(record.change, operation), + }; + const container = this.#containerKey(operation.container); + let history = this.#movableOrderHistory.get(container); + if (history === undefined) { + history = new OrderedIndex((left, right) => compareWriter(left.writer, right.writer)); + this.#movableOrderHistory.set(container, history); + } + history.add(indexed); + if (content.type === "movable-list-move") { + let peers = this.#movableMovePeers.get(container); + if (peers === undefined) { + peers = new Set(); + this.#movableMovePeers.set(container, peers); + } + peers.add(record.change.id.peer); + } + } + let bySubject; + let subject; + let treeOperation = false; + if (content.type === "map-insert" || content.type === "map-delete") { + const container = this.#containerKey(operation.container); + bySubject = this.#mapOperationHistory.get(container); + if (bySubject === undefined) { + bySubject = new Map(); + this.#mapOperationHistory.set(container, bySubject); + } + subject = content.key; + } + else if (content.type === "tree-create" || + content.type === "tree-move" || + content.type === "tree-delete") { + const container = this.#containerKey(operation.container); + bySubject = this.#treeOperationHistory.get(container); + if (bySubject === undefined) { + bySubject = new Map(); + this.#treeOperationHistory.set(container, bySubject); + } + subject = idKey(content.subject); + treeOperation = true; + } + if (bySubject === undefined || subject === undefined) + continue; + // Only movable/map/tree operations reach here, so the index entry and + // its writer are allocated off the text/list hot path. + const indexed = { + record, + operation, + writer: operationWriter(record.change, operation), + }; + let history = bySubject.get(subject); + if (history === undefined) { + history = { + byWriter: new OrderedIndex((left, right) => compareWriter(left.writer, right.writer)), + byPeer: new Map(), + ...(treeOperation ? { placementsByPeer: new Map() } : {}), + }; + bySubject.set(subject, history); + } + history.byWriter.add(indexed); + let peerOperations = history.byPeer.get(record.change.id.peer); + if (peerOperations === undefined) { + peerOperations = []; + history.byPeer.set(record.change.id.peer, peerOperations); + } + const peerIndex = lowerBoundIndexedOperation(peerOperations, operation.counter); + peerOperations.splice(peerIndex, 0, indexed); + if (content.type === "tree-create" || content.type === "tree-move") { + let placements = history.placementsByPeer.get(record.change.id.peer); + if (placements === undefined) { + placements = []; + history.placementsByPeer.set(record.change.id.peer, placements); + } + const placementIndex = lowerBoundIndexedOperation(placements, operation.counter); + placements.splice(placementIndex, 0, indexed); + } + } + } + #validateDeferredFrontierBlocks(entries, endVersion, frontiers) { + const changeEntriesByPeer = new Map(); + for (const entry of entries) { + if (entry.key.length !== 12) + continue; + const start = decodeChangeBlockKey(entry.key); + const peerEntries = changeEntriesByPeer.get(start.peer); + if (peerEntries === undefined) { + changeEntriesByPeer.set(start.peer, [{ entry, start }]); + } + else { + peerEntries.push({ entry, start }); + } + } + for (const peerEntries of changeEntriesByPeer.values()) { + peerEntries.sort((left, right) => left.start.counter - right.start.counter); + } + const seenPeers = new Set(); + for (const frontier of frontiers) { + const peerEnd = endVersion.get(frontier.peer) ?? 0; + if (frontier.counter >= peerEnd) { + throw new Error("snapshot frontier exceeds its end version"); + } + if (seenPeers.has(frontier.peer)) { + throw new Error("snapshot contains multiple frontiers for one peer"); + } + seenPeers.add(frontier.peer); + const peerEntries = changeEntriesByPeer.get(frontier.peer) ?? []; + let low = 0; + let high = peerEntries.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (peerEntries[middle].start.counter <= frontier.counter) + low = middle + 1; + else + high = middle; + } + const candidate = peerEntries[low - 1]; + if (candidate === undefined) { + throw new Error("snapshot frontier change block is missing"); + } + const block = validateChangeBlock(candidate.entry.value); + if (block.peer !== candidate.start.peer || + block.counterStart !== candidate.start.counter) { + throw new Error("snapshot change key does not match its block"); + } + if (block.peer !== frontier.peer || + frontier.counter < block.counterStart || + frontier.counter >= block.counterEnd) { + throw new Error("snapshot frontier is not covered by its change block"); + } + } + } + #materializeDeferredHistory() { + const deferred = this.#deferredSnapshotHistory; + if (deferred === undefined) + return; + const records = []; + for (const entry of deferred.entries) { + if (entry.key.length !== 12) + continue; + const expected = decodeChangeBlockKey(entry.key); + const decoded = this.#readChangeBlock(entry.value); + if (decoded[0] !== undefined && !idsEqual(decoded[0].change.id, expected)) { + throw new Error("snapshot change key does not match its block"); + } + records.push(...decoded.map(cloneHistoryRecord)); + } + // Validate and build every history index on an isolated document. A malformed + // deferred block must not leave this document with half-installed history. + const staged = new LoroDoc(); + staged.#shallowStartVersion = this.#shallowStartVersion.clone(); + const integration = staged.#integrateHistory(records); + if (integration.pending.length > 0) { + throw new Error("snapshot history contains changes with missing dependencies"); + } + if (staged.#historyVersion().compare(deferred.endVersion) !== 0) { + throw new Error("snapshot version does not match its history"); + } + const actualFrontiers = [...staged.#historyFrontiers.values()] + .sort(compareIds) + .map(formatOpId); + const expectedFrontiers = [...deferred.frontiers].sort(compareIds).map(formatOpId); + if (!frontierSetsEqual(actualFrontiers, expectedFrontiers)) { + throw new Error("snapshot frontiers do not match its history"); + } + if (staged.#historyOperationCount !== deferred.operationCount) { + throw new Error("snapshot operation count does not match its history"); + } + // State was already hydrated from the latest-state SSTable. Installing these + // structures restores only history/DAG indexes; applying records would + // duplicate counters and sequence content. + this.#history = staged.#history; + this.#historyOrder = staged.#historyOrder; + this.#historyByPeer = staged.#historyByPeer; + this.#historyEndByPeer = staged.#historyEndByPeer; + this.#historyOperationCount = staged.#historyOperationCount; + this.#sortedHistoryCache = staged.#sortedHistoryCache; + this.#historyFrontiers = staged.#historyFrontiers; + this.#dependencyVersionCache = staged.#dependencyVersionCache; + this.#mapOperationHistory = staged.#mapOperationHistory; + this.#treeOperationHistory = staged.#treeOperationHistory; + this.#movableOrderHistory = staged.#movableOrderHistory; + this.#movableMovePeers = staged.#movableMovePeers; + this.#containersWithOperations = staged.#containersWithOperations; + this.#containerKeys = staged.#containerKeys; + this.#pendingHistory = staged.#pendingHistory; + this.#deferredSnapshotHistory = undefined; + } + #sortedHistory() { + this.#materializeDeferredHistory(); + return (this.#sortedHistoryCache ??= this.#historyOrder.values()); + } + #recordsAtVersion(version) { + return this.#recordsInVersionRange(new VersionVector(), version); + } + #recordsInVersionRange(from, to) { + this.#materializeDeferredHistory(); + const selected = []; + for (const { peer, counter: toCounter } of to._codecEntriesUnsorted()) { + const fromCounter = from.get(peer) ?? 0; + if (fromCounter >= toCounter) + continue; + const records = this.#historyByPeer.get(peer) ?? []; + let index = Math.max(0, lowerBoundHistory(records, fromCounter + 1) - 1); + for (; index < records.length; index += 1) { + const record = records[index]; + const recordStart = record.change.id.counter; + if (recordStart >= toCounter) + break; + const length = changeLength(record.change); + const start = Math.max(0, fromCounter - recordStart); + const end = Math.min(length, toCounter - recordStart); + if (start >= end) + continue; + selected.push(start === 0 && end === length + ? record + : { ...record, change: sliceChange(record.change, start, end) }); + } + } + return selected.sort(compareHistoryRecords); + } + #recordsInSpans(spans) { + this.#materializeDeferredHistory(); + const spansByPeer = new Map(); + for (const { id, len } of spans) { + if (!Number.isSafeInteger(len) || len < 0) { + throw new RangeError(`update span length is out of range: ${len}`); + } + const parsed = parseOpId(id); + if (len === 0) + continue; + let peerSpans = spansByPeer.get(parsed.peer); + if (peerSpans === undefined) { + peerSpans = []; + spansByPeer.set(parsed.peer, peerSpans); + } + peerSpans.push({ start: parsed.counter, end: parsed.counter + len }); + } + const selected = []; + for (const [peer, ranges] of spansByPeer) { + ranges.sort((left, right) => left.start - right.start); + const merged = []; + for (const range of ranges) { + const previous = merged.length > 0 ? merged[merged.length - 1] : undefined; + if (previous !== undefined && range.start <= previous.end) { + previous.end = Math.max(previous.end, range.end); + } + else { + merged.push({ ...range }); + } + } + const records = this.#historyByPeer.get(peer) ?? []; + for (const range of merged) { + let index = Math.max(0, lowerBoundHistory(records, range.start + 1) - 1); + for (; index < records.length; index += 1) { + const record = records[index]; + const changeStart = record.change.id.counter; + if (changeStart >= range.end) + break; + const length = changeLength(record.change); + const start = Math.max(0, range.start - changeStart); + const end = Math.min(length, range.end - changeStart); + if (start >= end) + continue; + selected.push(start === 0 && end === length + ? record + : { ...record, change: sliceChange(record.change, start, end) }); + } + } + const preCommit = this.#preCommitRecord; + if (preCommit?.change.id.peer === peer) { + const changeStart = preCommit.change.id.counter; + const length = changeLength(preCommit.change); + const changeEnd = changeStart + length; + for (const range of merged) { + const start = Math.max(changeStart, range.start); + const end = Math.min(changeEnd, range.end); + if (start >= end) + continue; + selected.push(start === changeStart && end === changeEnd + ? preCommit + : { + ...preCommit, + change: sliceChange(preCommit.change, start - changeStart, end - changeStart), + }); + } + } + } + return selected.sort(compareHistoryRecords); + } + #assertVersionNotBeforeShallowRoot(version) { + if (!versionIncludes(version, this.#shallowRootVersion)) { + throw new RangeError("cannot use a version before the shallow history root"); + } + } + #causalVersionAt(frontiers) { + // Most documents have no shallow root; avoid the entry array and map + // copies in that case. + const version = this.#shallowStartVersion.length() === 0 + ? new Map() + : new Map(this.#shallowStartVersion + ._codecEntriesUnsorted() + .map(({ peer, counter }) => [peer, counter])); + for (const id of frontiers) { + const record = this.#recordContaining(id); + if (record !== undefined) { + for (const [peer, counter] of this.#dependencyVersion(record.change)) { + version.set(peer, Math.max(version.get(peer) ?? 0, counter)); + } + } + version.set(id.peer, Math.max(version.get(id.peer) ?? 0, id.counter + 1)); + } + return version; + } + #dependencyVersion(change) { + const cached = this.#dependencyVersionCache.get(change); + if (cached !== undefined) + return cached; + const version = this.#shallowStartVersion.length() === 0 + ? new Map() + : new Map(this.#shallowStartVersion + ._codecEntriesUnsorted() + .map(({ peer, counter }) => [peer, counter])); + for (const dependency of change.dependencies) { + const dependencyRecord = this.#recordContaining(dependency); + if (dependencyRecord !== undefined && dependencyRecord.change !== change) { + for (const [peer, counter] of this.#dependencyVersion(dependencyRecord.change)) { + version.set(peer, Math.max(version.get(peer) ?? 0, counter)); + } + } + version.set(dependency.peer, Math.max(version.get(dependency.peer) ?? 0, dependency.counter + 1)); + } + this.#dependencyVersionCache.set(change, version); + return version; + } + #frontiersForVersion(version) { + const candidates = new Map(); + const candidateByPeer = new Map(); + const known = this.#historyVersion(); + for (const { peer, counter } of version._codecEntriesUnsorted()) { + const end = Math.min(counter, known.get(peer) ?? 0); + if (end === 0) + continue; + const last = { peer, counter: end - 1 }; + if (this.#recordContaining(last) !== undefined) { + candidates.set(idKey(last), last); + candidateByPeer.set(peer, last); + } + } + for (const frontier of [...candidates.values()]) { + const causalVersion = this.#causalVersionAt([frontier]); + for (const [peer, counter] of causalVersion) { + const candidate = candidateByPeer.get(peer); + if (candidate !== undefined && + !idsEqual(candidate, frontier) && + candidate.counter < counter) { + candidates.delete(idKey(candidate)); + } + } + } + return [...candidates.values()].sort(compareIds); + } + #frontiersCodec() { + const candidates = new Map(); + if (this.#checkoutVersion !== undefined) { + for (const frontier of this.#frontiersForVersion(this.#checkoutVersion)) { + candidates.set(idKey(frontier), frontier); + } + } + else if (this.#deferredSnapshotHistory !== undefined) { + for (const frontier of this.#deferredSnapshotHistory.frontiers) { + candidates.set(idKey(frontier), frontier); + } + } + else { + for (const [key, frontier] of this.#historyFrontiers) { + candidates.set(key, frontier); + } + } + if (this.#pending !== undefined && this.#pending.operations.length > 0) { + for (const dependency of this.#pending.dependencies) + candidates.delete(idKey(dependency)); + const last = { + peer: this.#peer, + counter: this.#pending.id.counter + this.#pending.operationLength - 1, + }; + candidates.set(idKey(last), last); + } + return [...candidates.values()].sort(compareIds); + } + #lamportAt(id) { + const record = this.#recordContaining(id); + return record === undefined + ? 0 + : record.change.lamport + (id.counter - record.change.id.counter); + } + #encodeUpdates(records) { + const blocks = records.map((record) => encodeSingleChangeBlock(record.change, record.keys)); + return encodeDocument(EncodeMode.FastUpdates, encodeFastUpdatesBody(blocks)); + } + #encodeSnapshot() { + if (this.isShallow()) { + return this.#encodeShallowSnapshot(this.shallowSinceFrontiers()); + } + const historyEntries = this.#sortedHistory().map((record) => ({ + key: encodeChangeBlockKey(record.change.id), + value: encodeSingleChangeBlock(record.change, record.keys), + })); + historyEntries.push({ + key: VERSION_KEY, + value: encodePostcardVersionVector(this.version().codecEntries()), + }); + historyEntries.push({ + key: FRONTIERS_KEY, + value: encodePostcardFrontiers(this.#frontiersCodec()), + }); + const body = encodeFastSnapshotBody({ + oplog: encodeSstable(historyEntries, { compression: "auto" }), + state: encodeStateSnapshotStore(this.#buildStateStore(), { compression: "auto" }), + shallowRootState: new Uint8Array(), + }); + return encodeDocument(EncodeMode.FastSnapshot, body); + } + #encodeShallowSnapshot(requestedFrontiers) { + const startFrontiers = this.#calculateShallowStart(requestedFrontiers); + const rootVersion = this.#causalVersionForKnownFrontiers(startFrontiers); + const startVersion = rootVersion.clone(); + for (const frontier of startFrontiers) { + startVersion.set(frontier.peer, frontier.counter); + } + const latestVersion = this.#historyVersion(); + const latestFrontiers = this.#frontiersForVersion(latestVersion); + const restoreVersion = this.version(); + this.#rebuildFromHistory(rootVersion); + const rootStore = this.#buildStateStore(startFrontiers); + this.#rebuildFromHistory(latestVersion); + const latestStore = this.#buildStateStore(); + if (restoreVersion.compare(latestVersion) !== 0) { + this.#rebuildFromHistory(restoreVersion); + } + const historyEntries = this.#recordsInVersionRange(startVersion, latestVersion).map((record) => ({ + key: encodeChangeBlockKey(record.change.id), + value: encodeSingleChangeBlock(record.change, record.keys), + })); + historyEntries.push({ + key: VERSION_KEY, + value: encodePostcardVersionVector(latestVersion.codecEntries()), + }, { + key: FRONTIERS_KEY, + value: encodePostcardFrontiers(latestFrontiers), + }, { + key: START_VERSION_KEY, + value: encodePostcardVersionVector(startVersion.codecEntries()), + }, { + key: START_FRONTIERS_KEY, + value: encodePostcardFrontiers(startFrontiers), + }); + return encodeDocument(EncodeMode.FastSnapshot, encodeFastSnapshotBody({ + oplog: encodeSstable(historyEntries, { compression: "auto" }), + state: encodeStateSnapshotStore(latestStore, { compression: "auto" }), + shallowRootState: encodeStateSnapshotStore(rootStore, { + compression: "auto", + }), + })); + } + #calculateShallowStart(requested) { + const parsed = requested.map(parseOpId); + for (const frontier of requested) { + if (this.#recordContaining(parseOpId(frontier)) === undefined) { + throw new RangeError(`shallow snapshot frontier ${frontier.counter}@${frontier.peer} is unknown`); + } + } + let start = parsed; + if (start.length > 1) { + const versions = start.map((frontier) => this.#causalVersionAt([frontier])); + const peers = new Set(versions.flatMap((version) => [...version.keys()])); + const common = new VersionVector(); + for (const peer of peers) { + const counter = Math.min(...versions.map((version) => version.get(peer) ?? 0)); + if (counter > 0) + common.set(peer, counter); + } + const commonFrontiers = this.#frontiersForVersion(common); + start = commonFrontiers.length === 1 ? commonFrontiers : []; + } + if (start.length === 1) { + const operation = this.#operationAt(start[0]); + if (operation?.content.type === "text-mark") { + start = [{ peer: start[0].peer, counter: start[0].counter + 1 }]; + } + } + const candidateVersion = this.#causalVersionForKnownFrontiers(start); + if (this.isShallow() && + !versionIncludes(candidateVersion, this.#shallowRootVersion)) { + return this.#shallowRootFrontiers.map((id) => ({ ...id })); + } + return start; + } + #causalVersionForKnownFrontiers(frontiers) { + const version = new VersionVector(); + for (const [peer, counter] of this.#causalVersionAt(frontiers)) { + version.set(peer, counter); + } + return version; + } + #operationAt(id) { + const record = this.#recordContaining(id); + if (record === undefined) + return undefined; + let low = 0; + let high = record.change.operations.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (record.change.operations[middle].counter <= id.counter) + low = middle + 1; + else + high = middle; + } + const operation = low > 0 ? record.change.operations[low - 1] : undefined; + return operation !== undefined && id.counter < operation.counter + operation.length + ? operation + : undefined; + } + #buildStateStore(frontiers) { + const containers = []; + for (const container of this.#containers.values()) { + const id = container._codecId; + if (id === undefined) + continue; + const parent = container.parent()?._codecId; + containers.push({ + id, + wrapper: { + containerType: id.containerType, + depth: BigInt(containerDepth(container)), + parent, + state: this.#containerState(container), + }, + }); + } + return containers.length === 0 && frontiers === undefined + ? { kind: "empty" } + : { kind: "sstable", frontiers, containers }; + } + #containerState(container) { + if (container instanceof LoroMap) { + const peers = []; + const peerIndices = new Map(); + const peerIndex = (peer) => { + let index = peerIndices.get(peer); + if (index === undefined) { + index = peers.length; + peers.push(peer); + peerIndices.set(peer, index); + } + return BigInt(index); + }; + const values = []; + const deletedKeys = []; + const metadata = []; + for (const [key, record] of container._entries) { + if (record.deleted) + deletedKeys.push(key); + else + values.push([key, this.#encodeSnapshotValue(record.rawValue ?? record.value)]); + metadata.push({ + key, + peerIndex: peerIndex(record.writer.peer), + lamport: BigInt(record.writer.lamport), + }); + } + return { kind: CodecContainerType.Map, values, deletedKeys, peers, metadata }; + } + if (container instanceof LoroText) { + const peers = []; + const peerIndices = new Map(); + const peerIndex = (peer) => { + let index = peerIndices.get(peer); + if (index === undefined) { + index = peers.length; + peers.push(peer); + peerIndices.set(peer, index); + } + return BigInt(index); + }; + const keys = []; + const keyIndices = new Map(); + const keyIndex = (key) => { + let index = keyIndices.get(key); + if (index === undefined) { + index = keys.length; + keys.push(key); + keyIndices.set(key, index); + } + return index; + }; + const spans = []; + const marks = []; + const metasAt = container._attributeMetasResolver(); + let active = new Map(); + const updateActiveStyles = (current) => { + for (const [key, meta] of active) { + if (current.get(key) === meta) + continue; + spans.push({ + peerIndex: peerIndex(meta.startId.peer), + counter: meta.startId.counter + 1, + lamportSub: meta.lamport - meta.startId.counter, + length: -1, + }); + } + for (const [key, meta] of current) { + if (active.get(key) === meta) + continue; + spans.push({ + peerIndex: peerIndex(meta.startId.peer), + counter: meta.startId.counter, + lamportSub: meta.lamport - meta.startId.counter, + length: 0, + }); + marks.push({ + keyIndex: keyIndex(key), + value: this.#encodeSnapshotValue(meta.value), + info: meta.info, + }); + } + active = current; + }; + const textChunks = []; + let textChunk = ""; + let textChunkLength = 0; + const appendText = (text, scalarLength) => { + if (textChunkLength > 0 && textChunkLength + scalarLength > 1024) { + textChunks.push(textChunk); + textChunk = ""; + textChunkLength = 0; + } + textChunk += text; + textChunkLength += scalarLength; + if (textChunkLength === 1024) { + textChunks.push(textChunk); + textChunk = ""; + textChunkLength = 0; + } + }; + let cachedPeer; + let cachedPeerIndex = 0n; + const appendId = (peer, counter, lamport) => { + if (cachedPeer !== peer) { + cachedPeer = peer; + cachedPeerIndex = peerIndex(peer); + } + const lamportSub = lamport - counter; + const previous = spans[spans.length - 1]; + if (previous !== undefined && + previous.length > 0 && + previous.peerIndex === cachedPeerIndex && + previous.counter + previous.length === counter && + previous.lamportSub === lamportSub) { + previous.length += 1; + } + else { + spans.push({ + peerIndex: cachedPeerIndex, + counter, + lamportSub, + length: 1, + }); + } + }; + if (container._styleIndex.isEmpty) { + container._forEachVisibleSnapshotData(appendText, appendId); + } + else { + container._sequence.forEachVisible((element) => { + updateActiveStyles(metasAt(element)); + appendId(element.id.peer, element.id.counter, element.lamport); + appendText(element.value, 1); + }); + } + updateActiveStyles(new Map()); + if (textChunkLength > 0) + textChunks.push(textChunk); + return { + kind: CodecContainerType.Text, + text: textChunks.join(""), + peers, + spans, + keys, + marks, + }; + } + if (container instanceof LoroTree) { + // Rust's fast-tree decoder historically accepted the canonical snapshot + // order: live roots/subtrees first, then deleted roots/subtrees, with each + // sibling group ordered by fractional position and last-move writer. + // Preserve that order so snapshots remain byte-level interoperable and so + // parent-local ordered insertion never sees an insertion-order permutation. + const records = container._recordsForSnapshot(); + const peers = []; + const peerIndices = new Map(); + const peerIndex = (peer) => { + let index = peerIndices.get(peer); + if (index === undefined) { + index = peers.length; + peers.push(peer); + peerIndices.set(peer, index); + } + return BigInt(index); + }; + const positions = []; + const positionIndices = new Map(); + const recordIndices = new Map(records.map((record, index) => [idKey(record.id), index])); + return { + kind: CodecContainerType.Tree, + peers, + nodes: records.map((record) => { + const positionKey = bytesToHex(record.position); + let positionIndex = positionIndices.get(positionKey); + if (positionIndex === undefined) { + positionIndex = positions.length; + positions.push(record.position.slice()); + positionIndices.set(positionKey, positionIndex); + } + const parentIndex = record.deleted + ? 1n + : record.parent === undefined + ? 0n + : BigInt((recordIndices.get(idKey(record.parent)) ?? -2) + 2); + return { + peerIndex: peerIndex(record.id.peer), + counter: record.id.counter, + parentIndexPlusTwo: parentIndex, + lastSetPeerIndex: peerIndex(record.writer.peer), + lastSetCounter: record.lastMoveId.counter, + lastSetLamportSub: record.writer.lamport - record.lastMoveId.counter, + fractionalIndexIndex: positionIndex, + }; + }), + positions, + reserved: new Uint8Array(), + }; + } + if (container instanceof LoroCounter) { + float64BitsScratch.setFloat64(0, container.value, true); + return { + kind: CodecContainerType.Counter, + bits: float64BitsScratch.getBigUint64(0, true), + }; + } + if (!(container instanceof LoroList)) { + throw new TypeError(`unsupported container kind ${container.kind()}`); + } + const visible = container._visibleElements(); + const peers = []; + const peerIndices = new Map(); + const peerIndex = (peer) => { + let index = peerIndices.get(peer); + if (index === undefined) { + index = peers.length; + peers.push(peer); + peerIndices.set(peer, index); + } + return BigInt(index); + }; + if (container instanceof LoroMovableList) { + // The state-snapshot encoder only reads item fields, so every slot can + // reference the same frozen item instead of allocating one per element. + const items = [SHARED_MOVABLE_LIST_STATE_ITEM]; + for (let index = 0; index < visible.length; index += 1) { + items.push(SHARED_MOVABLE_LIST_STATE_ITEM); + } + return { + kind: CodecContainerType.MovableList, + values: visible.map((element) => this.#encodeSnapshotValue(element.value)), + peers, + items, + listItemIds: visible.map((element) => ({ + peerIndex: peerIndex(element.id.peer), + counter: element.id.counter, + lamportSub: element.lamport - element.id.counter, + })), + elementIds: [], + lastSetIds: [], + }; + } + return { + kind: CodecContainerType.List, + values: visible.map((element) => this.#encodeSnapshotValue(element.value)), + peers, + ids: visible.map((element) => ({ + peerIndex: peerIndex(element.id.peer), + counter: element.id.counter, + lamportSub: element.lamport - element.id.counter, + })), + }; + } + #encodeSnapshotValue(value) { + if (isContainer(value)) { + if (value._codecId === undefined) + throw new Error("detached child in attached state"); + return { type: "container", value: value._codecId }; + } + if (value === null) + return { type: "null" }; + if (typeof value === "boolean") + return { type: "bool", value }; + if (typeof value === "number") { + return Number.isSafeInteger(value) + ? { type: "i64", value: BigInt(value) } + : { type: "double", value }; + } + if (typeof value === "string") + return { type: "string", value }; + if (value instanceof Uint8Array) + return { type: "binary", value: value.slice() }; + if (Array.isArray(value)) + return { + type: "list", + value: value.map((item) => this.#encodeSnapshotValue(item)), + }; + return { + type: "map", + value: Object.entries(value).map(([key, item]) => [ + key, + this.#encodeSnapshotValue(item), + ]), + }; + } + #hydrateState(store, textSnapshotIds = validateStateSnapshotTextIds(store)) { + if (store.kind !== "sstable") + return; + let remainingTextCounterSlots = 10000000; + for (const { id } of store.containers) + this.#getOrCreateContainer(id); + for (const { id, wrapper } of store.containers) { + const container = this.#getOrCreateContainer(id); + const parent = wrapper.parent === undefined + ? undefined + : this.#getOrCreateContainer(wrapper.parent); + container._attach(this, id, parent); + container._reset(); + } + for (const { id, wrapper } of store.containers) { + const container = this.#getOrCreateContainer(id); + const state = wrapper.state; + if (container instanceof LoroMap && state.kind === CodecContainerType.Map) { + const metadata = new Map(state.metadata.map((item) => [item.key, item])); + for (const [key, value] of state.values) { + const item = metadata.get(key); + const rawValue = this.#decodeSnapshotValue(value, container); + container._applyValue(key, this.#materializeMapValue(container, key, rawValue), { + peer: state.peers[Number(item.peerIndex)], + lamport: Number(item.lamport), + }, rawValue); + } + for (const key of state.deletedKeys) { + const item = metadata.get(key); + container._applyDelete(key, { + peer: state.peers[Number(item.peerIndex)], + lamport: Number(item.lamport), + }); + } + } + else if (container instanceof LoroText && + state.kind === CodecContainerType.Text) { + const snapshotIds = textSnapshotIds.get(state); + for (const { peer, end, visible } of snapshotIds.reservations) { + if (end > 0 && end <= visible * 4 + 4096 && end <= remainingTextCounterSlots) { + container._sequence.reservePeerCounters(peer, end); + remainingTextCounterSlots -= end; + } + } + container._beginValidatedSnapshotLoad(); + const styleRuns = []; + const stylesById = new Map(); + const active = new Map(); + let textUtf16Offset = 0; + let chunkTextStart = 0; + const chunkPeers = []; + const chunkCounters = []; + const chunkLamports = []; + const flushChunk = () => { + if (chunkPeers.length === 0) + return; + container._insertValidatedSnapshotChunk(state.text.slice(chunkTextStart, textUtf16Offset), chunkPeers, chunkCounters, chunkLamports); + chunkPeers.length = 0; + chunkCounters.length = 0; + chunkLamports.length = 0; + chunkTextStart = textUtf16Offset; + }; + let markIndex = 0; + for (const span of state.spans) { + const peer = state.peers[Number(span.peerIndex)]; + if (span.length === 0) { + const mark = state.marks[markIndex++]; + const key = state.keys[mark.keyIndex]; + const value = this.#decodeSnapshotValue(mark.value); + const meta = { + startId: { peer, counter: span.counter }, + lamport: span.counter + span.lamportSub, + info: mark.info, + value, + }; + stylesById.set(idKey(meta.startId), { key, meta }); + const stack = active.get(key) ?? []; + stack.push(meta); + active.set(key, stack); + continue; + } + if (span.length === -1) { + const style = stylesById.get(idKey({ peer, counter: span.counter - 1 })); + if (style !== undefined) { + const stack = active.get(style.key); + if (stack !== undefined) { + const index = stack.lastIndexOf(style.meta); + if (index >= 0) + stack.splice(index, 1); + if (stack.length === 0) + active.delete(style.key); + } + } + continue; + } + if (span.length < -1) + continue; + for (const [key, stack] of active) { + const meta = stack.at(-1); + if (meta !== undefined) { + styleRuns.push({ + run: { start: { peer, counter: span.counter }, length: span.length }, + key, + meta, + }); + } + } + for (let offset = 0; offset < span.length; offset += 1) { + // Inline advanceUnicodeScalars(state.text, textUtf16Offset, 1). + if (textUtf16Offset >= state.text.length) { + throw new Error("text snapshot span exceeds the decoded text"); + } + const first = state.text.charCodeAt(textUtf16Offset); + textUtf16Offset += 1; + if (first >= 0xd800 && + first <= 0xdbff && + textUtf16Offset < state.text.length && + state.text.charCodeAt(textUtf16Offset) >= 0xdc00 && + state.text.charCodeAt(textUtf16Offset) <= 0xdfff) { + textUtf16Offset += 1; + } + chunkPeers.push(peer); + chunkCounters.push(span.counter + offset); + chunkLamports.push(span.counter + offset + span.lamportSub); + if (chunkPeers.length === 32) + flushChunk(); + } + } + flushChunk(); + if (textUtf16Offset !== state.text.length) { + throw new Error("text snapshot spans do not consume the decoded text"); + } + container._endValidatedSnapshotLoad(); + for (const { run, key, meta } of styleRuns) { + container._styleIndex.add([run], key, meta); + } + container._attributeHistoryComplete = false; + } + else if (container instanceof LoroTree && + state.kind === CodecContainerType.Tree) { + const records = []; + for (let index = 0; index < state.nodes.length; index += 1) { + const node = state.nodes[index]; + const nodeId = { + peer: state.peers[Number(node.peerIndex)], + counter: node.counter, + }; + const dataId = { + kind: "normal", + ...nodeId, + containerType: CodecContainerType.Map, + }; + const record = { + id: nodeId, + parent: undefined, + position: state.positions[node.fractionalIndexIndex].slice(), + deleted: node.parentIndexPlusTwo === 1n, + writer: { + peer: state.peers[Number(node.lastSetPeerIndex)], + lamport: node.lastSetCounter + node.lastSetLamportSub, + }, + lastMoveId: { + peer: state.peers[Number(node.lastSetPeerIndex)], + counter: node.lastSetCounter, + }, + data: this.#getOrCreateContainer(dataId, container), + }; + records.push(record); + } + for (let index = 0; index < state.nodes.length; index += 1) { + const parent = state.nodes[index].parentIndexPlusTwo; + if (parent >= 2n) + records[index].parent = records[Number(parent - 2n)].id; + } + for (const record of records) + container._setRecord(record); + } + else if (container instanceof LoroCounter && + state.kind === CodecContainerType.Counter) { + float64BitsScratch.setBigUint64(0, state.bits, true); + container._value = float64BitsScratch.getFloat64(0, true); + } + else if (container instanceof LoroMovableList && + state.kind === CodecContainerType.MovableList) { + const ids = state.listItemIds.slice(0, state.values.length); + container._insertVisible(0, state.values.map((value) => this.#decodeSnapshotValue(value, container)), ids.map((item) => ({ + peer: state.peers[Number(item.peerIndex)], + counter: item.counter, + })), ids.map((item) => item.counter + item.lamportSub)); + container._valueHistoryComplete = false; + container._moveHistoryComplete = false; + } + else if (container instanceof LoroList && + state.kind === CodecContainerType.List) { + container._insertVisible(0, state.values.map((value) => this.#decodeSnapshotValue(value, container)), state.ids.map((item) => ({ + peer: state.peers[Number(item.peerIndex)], + counter: item.counter, + })), state.ids.map((item) => item.counter + item.lamportSub)); + } + } + } + #decodeSnapshotValue(value, parent) { + switch (value.type) { + case "null": + return null; + case "bool": + return value.value; + case "double": + return value.value; + case "i64": + return Number(value.value); + case "string": + return value.value; + case "binary": + return value.value.slice(); + case "list": + return value.value.map((item) => this.#decodeSnapshotValue(item)); + case "map": + return Object.fromEntries(value.value.map(([key, item]) => [key, this.#decodeSnapshotValue(item)])); + case "container": + return this.#getOrCreateContainer(value.value, parent); + } + } + #materializeMapValue(parent, key, rawValue) { + const parentId = parent._codecId; + if (parentId === undefined) + return rawValue; + const childType = parseMergeableMarker(parentId, key, rawValue); + if (childType === undefined) + return rawValue; + return this.#getOrCreateContainer(newMergeableContainerId(parentId, key, childType), parent); + } + #captureEventValues(records) { + const values = new Map(); + for (const { change } of records) { + for (const operation of change.operations) { + const id = this.#containerKey(operation.container); + if (values.has(id)) + continue; + const container = this.#containers.get(id); + values.set(id, container === undefined ? undefined : containerEventValue(container)); + } + } + return values; + } + #recordConcurrentTreeSubjects(recording, added, beforeVersion) { + const incomingVersion = new VersionVector(); + for (const { change } of added) { + const length = changeLength(change); + if (length === 0) + continue; + const frontier = { + peer: change.id.peer, + counter: change.id.counter + length - 1, + }; + for (const [peer, counter] of this.#causalVersionAt([frontier])) { + if (counter > (incomingVersion.get(peer) ?? 0)) { + incomingVersion.set(peer, counter); + } + } + } + // Rust calculates a concurrent import diff by retreating to the common + // ancestor and replaying both branches. Tree operations from the local-only + // branch can therefore produce visible reindex moves even though those + // operations are not present in the imported update. Capture those subjects + // so the incremental TS merge reports the same net transition. + for (const { change } of this.#recordsInVersionRange(incomingVersion, beforeVersion)) { + for (const operation of change.operations) { + const content = operation.content; + if (content.type !== "tree-create" && + content.type !== "tree-move" && + content.type !== "tree-delete") { + continue; + } + const container = this.#containers.get(this.#containerKey(operation.container)); + if (!(container instanceof LoroTree)) + continue; + const state = this.#treeEventState(recording, container); + const nodeId = formatTreeId(content.subject); + if (state.originals.has(nodeId)) + continue; + const record = container._nodes.get(nodeId); + state.originals.set(nodeId, record === undefined || container._isRecordDeleted(record) + ? undefined + : this.#treeEventNode(container, record)); + } + } + } + #captureContainerEventValues(ids) { + const values = new Map(); + for (const id of ids) { + const container = this.#containers.get(id); + values.set(id, container === undefined ? undefined : containerEventValue(container)); + } + return values; + } + #captureMapKeys(ids) { + const keys = new Map(); + for (const id of ids) { + const container = this.#containers.get(id); + if (container instanceof LoroMap) + keys.set(id, new Set(container._entries.keys())); + } + return keys; + } + #emit(by, origin, from, to, changed, beforeValues = new Map(), preparedDiffs = new Map()) { + if (changed.size === 0 || !this.#hasEventSubscribers()) + return; + const expandedChanged = new Set(changed); + const revived = new Set(); + const calculatedDiffs = new Map(preparedDiffs); + for (const id of changed) { + const container = this.#containers.get(id); + if (!(container instanceof LoroTree) || this._isContainerDeleted(container)) { + continue; + } + const diff = calculatedDiffs.get(id) ?? containerDiff(container, beforeValues.get(id)); + calculatedDiffs.set(id, diff); + if (diff.type !== "tree") + continue; + for (const item of diff.diff) { + if (item.action !== "create") + continue; + const record = container._nodes.get(item.target); + if (record === undefined || container._isRecordDeleted(record)) + continue; + expandedChanged.add(record.data.id); + revived.add(record.data.id); + } + } + const events = []; + for (const id of expandedChanged) { + const container = this.#containers.get(id); + if (container === undefined || this._isContainerDeleted(container)) + continue; + const diff = calculatedDiffs.get(id) ?? + containerDiff(container, revived.has(id) ? undefined : beforeValues.get(id)); + if (isEmptyContainerDiff(diff)) + continue; + events.push({ target: id, diff, path: containerPath(container) }); + } + if (events.length === 0) + return; + const fromFormatted = from.map(formatOpId); + const toFormatted = to.map(formatOpId); + // Two explicit literals: the key set differs by `origin`, so each branch + // keeps one stable object shape and avoids CopyDataProperties. + const base = (origin === undefined + ? { by, events, from: fromFormatted, to: toFormatted } + : { by, origin, events, from: fromFormatted, to: toFormatted }); + for (const listener of this.#subscribers) + listener(base); + const relevantByTarget = new Map(); + for (const event of events) { + let current = this.#containers.get(event.target); + while (current !== undefined) { + const target = current.id; + if (this.#containerSubscribers.has(target)) { + const relevant = relevantByTarget.get(target); + if (relevant === undefined) + relevantByTarget.set(target, [event]); + else + relevant.push(event); + } + current = current.parent(); + } + } + for (const [target, relevant] of relevantByTarget) { + const listeners = this.#containerSubscribers.get(target); + if (listeners === undefined) + continue; + const batch = origin === undefined + ? { + by, + events: relevant, + from: fromFormatted, + to: toFormatted, + currentTarget: target, + } + : { + by, + origin, + events: relevant, + from: fromFormatted, + to: toFormatted, + currentTarget: target, + }; + for (const listener of listeners) + listener(batch); + } + } + #hasEventSubscribers() { + return this.#subscribers.size > 0 || this.#containerSubscribers.size > 0; + } +} +export class ChangeModifier { + #options; + constructor(options) { + this.#options = options; + } + free() { } + setMessage(message) { + this.#options.message = message; + return this; + } + setTimestamp(timestamp) { + if (!Number.isFinite(timestamp)) + throw new TypeError("timestamp must be finite"); + this.#options.timestamp = timestamp; + return this; + } +} +export class Loro extends LoroDoc { +} +export function callPendingEvents() { } +export function LORO_VERSION() { + return packageMetadata.version; +} +export function encodeFrontiers(frontiers) { + return encodePostcardFrontiers(frontiers.map(parseOpId)); +} +export function decodeFrontiers(bytes) { + return decodePostcardFrontiers(bytes).map(formatOpId); +} +export function setDebug() { } +export function decodeImportBlobMeta(blob, checkChecksum = true) { + const parsed = decodeDocument(blob, { checkChecksum }); + if (parsed.mode === EncodeMode.FastUpdates) { + const records = decodeFastUpdatesBody(parsed.body).flatMap(decodeHistoryRecordBlock); + const startVersion = new VersionVector(); + const endVersion = new VersionVector(); + for (const { change } of records) { + const currentStart = startVersion.get(change.id.peer); + if (currentStart === undefined || change.id.counter < currentStart) { + startVersion.set(change.id.peer, change.id.counter); + } + const end = change.id.counter + changeLength(change); + if (end > (endVersion.get(change.id.peer) ?? 0)) { + endVersion.set(change.id.peer, end); + } + } + const startFrontiers = new Map(); + for (const { change } of records) { + for (const dependency of change.dependencies) { + const start = startVersion.get(dependency.peer); + if ((start !== undefined && start > dependency.counter) || + (start === undefined && endVersion.get(dependency.peer) === undefined)) { + startFrontiers.set(idKey(dependency), dependency); + } + } + } + return { + partialStartVersionVector: startVersion, + partialEndVersionVector: endVersion, + startFrontiers: [...startFrontiers.values()].map(formatOpId), + startTimestamp: Number(records[0]?.change.timestamp ?? 0n), + endTimestamp: Number(records.at(-1)?.change.timestamp ?? 0n), + mode: "update", + changeNum: records.length, + }; + } + const snapshot = decodeFastSnapshotBody(parsed.body); + const records = []; + let startVersion = new VersionVector(); + let endVersion = new VersionVector(); + let startFrontiers = []; + let endFrontiers = []; + for (const entry of decodeSstable(snapshot.oplog)) { + if (bytesEqual(entry.key, VERSION_KEY)) { + endVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); + } + else if (bytesEqual(entry.key, FRONTIERS_KEY)) { + endFrontiers = decodePostcardFrontiers(entry.value); + } + else if (bytesEqual(entry.key, START_VERSION_KEY)) { + startVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); + } + else if (bytesEqual(entry.key, START_FRONTIERS_KEY)) { + startFrontiers = decodePostcardFrontiers(entry.value); + } + else if (entry.key.length === 12) { + records.push(...decodeHistoryRecordBlock(entry.value)); + } + } + if (endVersion.length() === 0) { + for (const { change } of records) { + const end = change.id.counter + changeLength(change); + if (end > (endVersion.get(change.id.peer) ?? 0)) + endVersion.set(change.id.peer, end); + } + } + const timestampAt = (frontiers) => { + const targetByPeer = new Map(frontiers.map((id) => [id.peer, id])); + let timestamp = 0; + for (const { change } of records) { + const frontier = targetByPeer.get(change.id.peer); + if (frontier === undefined) + continue; + const end = change.id.counter + changeLength(change); + if (frontier.counter >= change.id.counter && frontier.counter < end) { + timestamp = Math.max(timestamp, Number(change.timestamp)); + } + } + return timestamp; + }; + let latestRecord; + for (const record of records) { + if (latestRecord === undefined || compareHistoryRecords(latestRecord, record) < 0) { + latestRecord = record; + } + } + return { + partialStartVersionVector: startVersion, + partialEndVersionVector: endVersion, + startFrontiers: startFrontiers.map(formatOpId), + startTimestamp: timestampAt(startFrontiers), + endTimestamp: timestampAt(endFrontiers) || Number(latestRecord?.change.timestamp ?? 0n), + mode: snapshot.shallowRootState.length > 0 ? "shallow-snapshot" : "snapshot", + changeNum: records.length, + }; +} +function decodeHistoryRecordBlock(bytes) { + const block = decodeChangeBlock(bytes); + return block.changes.map((change) => ({ + change, + keys: block.keys, + keyIndices: undefined, + })); +} +function versionVectorFromCodec(ids) { + return new VersionVector(new Map(ids.map((id) => [peerIdToString(id.peer), id.counter]))); +} +function diffContainerId(value) { + if (isContainer(value)) + return value.id; + if (typeof value !== "string" || !value.startsWith("🦜:")) + return undefined; + const id = value.slice("🦜:".length); + return isContainerId(id) ? id : undefined; +} +function ensureMergeableChild(parent, key, type) { + switch (type) { + case "Map": + return parent.ensureMergeableMap(key); + case "List": + return parent.ensureMergeableList(key); + case "MovableList": + return parent.ensureMergeableMovableList(key); + case "Text": + return parent.ensureMergeableText(key); + case "Tree": + return parent.ensureMergeableTree(key); + case "Counter": + return parent.ensureMergeableCounter(key); + } +} +function diffKindMismatch(container, diffType) { + return new TypeError(`cannot apply a ${diffType} diff to a ${container.kind()} container`); +} +function createContainer(type) { + switch (type) { + case "Map": + return new LoroMap(); + case "List": + return new LoroList(); + case "Text": + return new LoroText(); + case "Tree": + return new LoroTree(); + case "MovableList": + return new LoroMovableList(); + case "Counter": + return new LoroCounter(); + } +} +function registerKey(pending, key) { + const existing = pending.keyIndices.get(key); + if (existing !== undefined) + return existing; + const index = pending.keys.length; + pending.keys.push(key); + pending.keyIndices.set(key, index); + return index; +} +function changeKey(id) { + return `${id.peer}:${id.counter}`; +} +function idKey(id) { + return `${id.peer}:${id.counter}`; +} +function frontierSetsEqual(left, right) { + if (left.length !== right.length) + return false; + const rightIds = new Set(right.map((id) => idKey(parseOpId(id)))); + return left.every((id) => rightIds.has(idKey(parseOpId(id)))); +} +const changeLengthCache = new WeakMap(); +function changeLength(change) { + const cached = changeLengthCache.get(change); + if (cached !== undefined) + return cached; + const length = change.operations.reduce((sum, operation) => sum + operation.length, 0); + changeLengthCache.set(change, length); + return length; +} +function validateStateSnapshotTextIds(store) { + const textSnapshotIds = new Map(); + if (store.kind !== "sstable") + return textSnapshotIds; + for (const { wrapper } of store.containers) { + if (wrapper.state.kind === CodecContainerType.Text) { + textSnapshotIds.set(wrapper.state, validateTextSnapshotIdRanges(wrapper.state.peers, wrapper.state.spans)); + } + } + return textSnapshotIds; +} +function validateTextSnapshotIdRanges(peers, spans) { + const statsByPeer = new Map(); + for (const span of spans) { + if (span.length <= 0) + continue; + const peerIndex = Number(span.peerIndex); + const end = span.counter + span.length; + if (!Number.isSafeInteger(peerIndex) || + peerIndex < 0 || + peerIndex >= peers.length || + !Number.isSafeInteger(span.counter) || + span.counter < 0 || + !Number.isSafeInteger(end) || + end <= span.counter || + end > 2147483647) { + throw new Error("text snapshot contains an invalid ID range"); + } + const peer = peers[peerIndex]; + let stats = statsByPeer.get(peer); + if (stats === undefined) { + stats = { peer, count: 0, end: 0, visible: 0, offset: 0 }; + statsByPeer.set(peer, stats); + } + stats.count += 1; + stats.visible += span.length; + if (!Number.isSafeInteger(stats.visible)) { + throw new Error("text snapshot contains too many IDs for one peer"); + } + if (end > stats.end) + stats.end = end; + } + for (const stats of statsByPeer.values()) { + stats.ranges = new BigUint64Array(stats.count); + } + for (const span of spans) { + if (span.length <= 0) + continue; + const peerIndex = Number(span.peerIndex); + const end = span.counter + span.length; + const stats = statsByPeer.get(peers[peerIndex]); + stats.ranges[stats.offset] = (BigInt(span.counter) << 32n) | BigInt(end); + stats.offset += 1; + } + for (const stats of statsByPeer.values()) { + const peerRanges = stats.ranges; + peerRanges.sort(); + let previousEnd = -1; + for (const packed of peerRanges) { + const start = Number(packed >> 32n); + const end = Number(packed & 0xffffffffn); + if (start < previousEnd) { + throw new Error("text snapshot contains duplicate sequence IDs"); + } + previousEnd = end; + } + } + return { + reservations: [...statsByPeer.values()].map(({ peer, end, visible }) => ({ + peer, + end, + visible, + })), + }; +} +function changedContainerIds(records) { + return new Set(records.flatMap(({ change }) => change.operations.map((operation) => formatContainerId(operation.container)))); +} +function longestIncreasingSubsequenceIndices(values) { + if (values.length === 0) + return new Set(); + const tails = []; + const previous = new Int32Array(values.length); + previous.fill(-1); + for (let index = 0; index < values.length; index += 1) { + let low = 0; + let high = tails.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (values[tails[middle]] < values[index]) + low = middle + 1; + else + high = middle; + } + if (low > 0) + previous[index] = tails[low - 1]; + tails[low] = index; + } + const indices = new Set(); + let index = tails.at(-1); + while (index >= 0) { + indices.add(index); + index = previous[index]; + } + return indices; +} +function movableMoveTransitionMode(retreat, forward, movePeers) { + const changedMoveContainers = new Set(); + for (const { change } of [...retreat, ...forward]) { + for (const operation of change.operations) { + if (operation.content.type === "movable-list-move") { + changedMoveContainers.add(formatContainerId(operation.container)); + } + } + } + if (changedMoveContainers.size === 0) + return "anchors"; + if (retreat.length > 0 && forward.length > 0) + return "replay"; + for (const containerId of changedMoveContainers) { + if ((movePeers.get(containerId)?.size ?? 0) > 1) + return "replay"; + } + return "anchors"; +} +function hasMaterializedSequenceInsertions(records, containers) { + for (const { change } of records) { + for (const operation of change.operations) { + const content = operation.content; + if (content.type !== "text-insert" && + content.type !== "list-insert" && + content.type !== "movable-list-insert") { + continue; + } + const container = containers.get(formatContainerId(operation.container)); + if ((container instanceof LoroText || container instanceof LoroList) && + container._sequence.findById({ + peer: change.id.peer, + counter: operation.counter, + }) !== undefined) { + return true; + } + } + } + return false; +} +function canMergeChanges(left, right, interval) { + return (right.id.peer === left.id.peer && + right.id.counter === left.id.counter + changeLength(left) && + right.dependencies.length === 1 && + right.dependencies[0].peer === left.id.peer && + right.timestamp - left.timestamp <= interval && + right.message === left.message); +} +function appendHistoryRecord(left, right, leftLength) { + let keyIndices = left.keyIndices; + let keys; + if (keyIndices === undefined) { + keys = [...left.keys]; + keyIndices = new Map(); + for (const [index, key] of keys.entries()) { + if (!keyIndices.has(key)) + keyIndices.set(key, index); + } + left.keys = keys; + left.keyIndices = keyIndices; + } + else { + keys = left.keys; + } + // A change with no keys of its own needs no remapping, so its operations + // array can be reused as-is; nothing downstream mutates it. + let appendedOperations; + if (right.keys.length === 0) { + appendedOperations = right.change.operations; + } + else { + const remappedKeyIndices = right.keys.map((key) => { + const existing = keyIndices.get(key); + if (existing !== undefined) + return existing; + const index = keys.length; + keys.push(key); + keyIndices.set(key, index); + return index; + }); + appendedOperations = right.change.operations.map((operation) => remapOperationKeys(operation, remappedKeyIndices)); + } + const mergedOperations = left.change.operations; + for (const operation of appendedOperations) + mergedOperations.push(operation); + const rightLength = changeLength(right.change); + changeLengthCache.set(left.change, leftLength + rightLength); + const appended = { + change: { + ...right.change, + timestamp: left.change.timestamp, + message: left.change.message, + operations: appendedOperations, + }, + keys, + // Deliberately not the merged map: it is shared with `left` and keeps + // growing, and this slice record never merges further. + keyIndices: undefined, + }; + changeLengthCache.set(appended.change, rightLength); + return appended; +} +function cloneHistoryRecord(record) { + return { + change: { ...record.change, operations: [...record.change.operations] }, + keys: record.keys, + keyIndices: undefined, + }; +} +function appendToTrailingListInsert(operations, operation) { + const content = operation.content; + if (content.type !== "list-insert" && content.type !== "movable-list-insert") { + return false; + } + const previousIndex = operations.length - 1; + const previous = operations[previousIndex]; + if (previous === undefined || + previous.content.type !== content.type || + !containerIdsEqual(previous.container, operation.container) || + previous.counter + previous.length !== operation.counter || + previous.content.position + previous.length !== content.position) { + return false; + } + const values = previous.content.values; + for (const value of content.values) + values.push(value); + operations[previousIndex] = { + ...previous, + length: previous.length + operation.length, + content: { ...previous.content, values }, + }; + return true; +} +function remapOperationKeys(operation, remap) { + const content = operation.content; + switch (content.type) { + case "map-insert": + case "text-mark": + case "movable-list-set": + return { + ...operation, + content: { ...content, value: remapLoroValueKeys(content.value, remap) }, + }; + case "list-insert": + case "movable-list-insert": + return { + ...operation, + content: { + ...content, + values: content.values.map((value) => remapLoroValueKeys(value, remap)), + }, + }; + case "future": + return { + ...operation, + content: { ...content, value: remapChangeValueKeys(content.value, remap) }, + }; + default: + return operation; + } +} +function remapChangeValueKeys(value, remap) { + switch (value.type) { + case "loro-value": + return { ...value, value: remapLoroValueKeys(value.value, remap) }; + case "mark-start": + return { + ...value, + keyIndex: remapKeyIndex(value.keyIndex, remap), + value: remapLoroValueKeys(value.value, remap), + }; + case "list-set": + return { ...value, value: remapLoroValueKeys(value.value, remap) }; + default: + return value; + } +} +function remapLoroValueKeys(value, remap) { + if (value.type === "list") { + return { + ...value, + value: value.value.map((item) => remapLoroValueKeys(item, remap)), + }; + } + if (value.type === "map") { + return { + ...value, + value: value.value.map(([keyIndex, item]) => [ + remapKeyIndex(keyIndex, remap), + remapLoroValueKeys(item, remap), + ]), + }; + } + return value; +} +function remapKeyIndex(keyIndex, remap) { + if (keyIndex < 0n || keyIndex > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error("change value map key index is out of range"); + } + const mapped = remap[Number(keyIndex)]; + if (mapped === undefined) { + throw new Error("change value map key index is out of range"); + } + return BigInt(mapped); +} +const I64_MAX = 0x7fffffffffffffffn; +const I64_MIN = -0x8000000000000000n; +function numberToI64(value) { + if (Number.isNaN(value)) + return 0n; + if (value >= Number(I64_MAX)) + return I64_MAX; + if (value <= Number(I64_MIN)) + return I64_MIN; + return BigInt(Math.trunc(value)); +} +function maxBigInt(left, right) { + return left >= right ? left : right; +} +function sliceChange(change, from, to) { + const totalLength = changeLength(change); + if (!Number.isSafeInteger(from) || + !Number.isSafeInteger(to) || + from < 0 || + from >= to || + to > totalLength) { + throw new RangeError("change slice is out of range"); + } + const operations = []; + let offset = 0; + for (const operation of change.operations) { + const operationStart = offset; + const operationEnd = offset + operation.length; + offset = operationEnd; + if (operationEnd <= from) + continue; + if (operationStart >= to) + break; + const sliceFrom = Math.max(0, from - operationStart); + const sliceTo = Math.min(operation.length, to - operationStart); + if (sliceFrom === 0 && sliceTo === operation.length) { + operations.push(operation); + } + else { + operations.push(sliceOperation(operation, sliceFrom, sliceTo)); + } + } + return { + ...change, + id: { peer: change.id.peer, counter: change.id.counter + from }, + lamport: change.lamport + from, + dependencies: from === 0 + ? change.dependencies + : [{ peer: change.id.peer, counter: change.id.counter + from - 1 }], + operations, + }; +} +// Slice a string by Unicode scalar offsets without materializing per-scalar +// strings. Out-of-range offsets clamp like Array.from(value).slice would. +function sliceUnicodeScalars(value, from, to) { + let offset = 0; + let scalar = 0; + while (scalar < from && offset < value.length) { + const first = value.charCodeAt(offset); + offset += + first >= 0xd800 && + first <= 0xdbff && + offset + 1 < value.length && + value.charCodeAt(offset + 1) >= 0xdc00 && + value.charCodeAt(offset + 1) <= 0xdfff + ? 2 + : 1; + scalar += 1; + } + const start = offset; + while (scalar < to && offset < value.length) { + const first = value.charCodeAt(offset); + offset += + first >= 0xd800 && + first <= 0xdbff && + offset + 1 < value.length && + value.charCodeAt(offset + 1) >= 0xdc00 && + value.charCodeAt(offset + 1) <= 0xdfff + ? 2 + : 1; + scalar += 1; + } + return value.slice(start, offset); +} +function sliceOperation(operation, from, to) { + if (from < 0 || from >= to || to > operation.length) { + throw new RangeError("partial operation length is out of range"); + } + const length = to - from; + const content = operation.content; + switch (content.type) { + case "text-insert": + return { + ...operation, + counter: operation.counter + from, + length, + content: { + ...content, + position: content.position + from, + value: sliceUnicodeScalars(content.value, from, to), + }, + }; + case "list-insert": + case "movable-list-insert": + return { + ...operation, + counter: operation.counter + from, + length, + content: { + ...content, + position: content.position + from, + values: content.values.slice(from, to), + }, + }; + case "text-delete": + case "list-delete": + case "movable-list-delete": { + const signedLength = Number(content.length); + const positive = signedLength >= 0; + return { + ...operation, + counter: operation.counter + from, + length, + content: { + ...content, + position: positive ? content.position : content.position - from, + length: BigInt(positive ? length : -length), + startId: { + peer: content.startId.peer, + counter: content.startId.counter + (positive ? from : operation.length - to), + }, + }, + }; + } + default: + throw new Error(`cannot slice ${content.type} operation`); + } +} +function versionIncludes(version, required) { + return required + ._codecEntriesUnsorted() + .every(({ peer, counter }) => (version.get(peer) ?? 0) >= counter); +} +function historyVersionForRecords(records, base) { + const version = base?.clone() ?? new VersionVector(); + for (const { change } of records) { + const end = change.id.counter + changeLength(change); + if (end > (version.get(change.id.peer) ?? 0)) { + version.set(change.id.peer, end); + } + } + return version; +} +function versionDistance(start, end) { + let distance = 0; + for (const { peer, counter } of end._codecEntriesUnsorted()) { + distance += Math.max(0, counter - (start.get(peer) ?? 0)); + } + return distance; +} +function mergeStateSnapshotStores(root, overlay) { + if (root.kind !== "sstable") + return overlay; + if (overlay.kind !== "sstable") { + return { kind: "sstable", frontiers: undefined, containers: root.containers }; + } + const containers = new Map(root.containers.map((entry) => [formatContainerId(entry.id), entry])); + for (const entry of overlay.containers) { + containers.set(formatContainerId(entry.id), entry); + } + return { + kind: "sstable", + frontiers: undefined, + containers: [...containers.values()], + }; +} +function compareIds(left, right) { + return left.peer < right.peer + ? -1 + : left.peer > right.peer + ? 1 + : left.counter - right.counter; +} +function compareHistoryRecords(left, right) { + return (left.change.lamport - right.change.lamport || + compareIds(left.change.id, right.change.id)); +} +function compareHistoryOperations(leftChange, leftOperation, rightChange, rightOperation) { + return compareWriter(operationWriter(leftChange, leftOperation), operationWriter(rightChange, rightOperation)); +} +function lowerBoundHistory(records, counter) { + let low = 0; + let high = records.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (records[middle].change.id.counter < counter) + low = middle + 1; + else + high = middle; + } + return low; +} +function lowerBoundOperation(operations, counter) { + let low = 0; + let high = operations.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (operations[middle].counter < counter) + low = middle + 1; + else + high = middle; + } + return low; +} +function lowerBoundIndexedOperation(operations, counter) { + let low = 0; + let high = operations.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (operations[middle].operation.counter < counter) + low = middle + 1; + else + high = middle; + } + return low; +} +function lowerBoundWriter(operations, writer) { + let low = 0; + let high = operations.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compareWriter(operations[middle].writer, writer) < 0) + low = middle + 1; + else + high = middle; + } + return low; +} +function latestIncludedOperation(history, version) { + if (history === undefined) + return undefined; + let latest; + for (const [peer, operations] of history.byPeer) { + const index = lowerBoundIndexedOperation(operations, version.get(peer) ?? 0) - 1; + const operation = operations[index]; + if (operation !== undefined && + (latest === undefined || compareWriter(latest.writer, operation.writer) < 0)) { + latest = operation; + } + } + return latest; +} +function latestIncludedTreePlacement(history, version, before) { + if (history?.placementsByPeer === undefined) + return undefined; + let latest; + for (const [peer, operations] of history.placementsByPeer) { + const index = Math.min(lowerBoundIndexedOperation(operations, version.get(peer) ?? 0), lowerBoundWriter(operations, before)) - 1; + const operation = operations[index]; + if (operation !== undefined && + (latest === undefined || compareWriter(latest.writer, operation.writer) < 0)) { + latest = operation; + } + } + return latest; +} +function latestIncludedSequenceValue(history, version) { + if (history === undefined) + return undefined; + for (let index = history.length - 1; index >= 0; index -= 1) { + const meta = history[index]; + if (meta.id.counter < (version.get(meta.id.peer) ?? 0)) + return meta; + } + return undefined; +} +function hasSequenceValueMeta(history, writer, peer, counter) { + if (history === undefined) + return false; + let low = 0; + let high = history.length; + while (low < high) { + const middle = (low + high) >>> 1; + const meta = history[middle]; + if (compareWriter({ peer: meta.id.peer, lamport: meta.lamport }, writer) < 0) { + low = middle + 1; + } + else { + high = middle; + } + } + const meta = history[low]; + return meta?.id.peer === peer && meta.id.counter === counter; +} +function findSequenceMoveMeta(history, writer) { + const index = findSequenceMoveMetaIndex(history, writer); + return index < 0 ? undefined : history[index]; +} +function findSequenceMoveMetaIndex(history, writer) { + if (history === undefined) + return -1; + let low = 0; + let high = history.length; + while (low < high) { + const middle = (low + high) >>> 1; + const meta = history[middle]; + if (compareWriter({ peer: meta.id.peer, lamport: meta.lamport }, writer) < 0) { + low = middle + 1; + } + else { + high = middle; + } + } + return low < history.length ? low : -1; +} +function counterDelta(content) { + if (content.value.type === "double") + return content.value.value; + if (content.value.type === "i64") + return Number(content.value.value); + if (content.value.type === "delta-int") + return content.value.value; + return undefined; +} +function publicChange(change) { + return { + peer: peerIdToString(change.id.peer), + counter: change.id.counter, + lamport: change.lamport, + length: changeLength(change), + timestamp: Number(change.timestamp), + deps: change.dependencies.map(formatOpId), + message: change.message, + }; +} +function historyRecordsToJsonSchema(records, startVersion, withPeerCompression, nullMessage = true) { + const peers = collectJsonPeers(records, startVersion); + const peerMap = withPeerCompression + ? new Map(peers.map((peer, index) => [peer, BigInt(index)])) + : undefined; + return { + schema_version: 1, + start_version: Object.fromEntries(startVersion + .codecEntries() + .map(({ peer, counter }) => [(peerMap?.get(peer) ?? peer).toString(), counter])), + peers: withPeerCompression ? peers.map(peerIdToString) : null, + changes: records.map((record) => historyRecordToJsonChange(record, peerMap, nullMessage)), + }; +} +function historyRecordToJsonChange(record, peerMap, nullMessage) { + const { change } = record; + return { + id: formatJsonOpId(change.id, peerMap), + timestamp: Number(change.timestamp), + deps: change.dependencies.map((id) => formatJsonOpId(id, peerMap)), + lamport: change.lamport, + msg: change.message ?? (nullMessage ? null : undefined), + ops: change.operations.map((operation) => decodedOperationToJson(operation, record.keys, change.id.peer, peerMap)), + }; +} +function collectJsonPeers(records, startVersion) { + const peers = new Set(startVersion._codecEntriesUnsorted().map(({ peer }) => peer)); + const addId = (id) => { + if (id !== undefined) + peers.add(id.peer); + }; + for (const { change } of records) { + addId(change.id); + for (const dependency of change.dependencies) + addId(dependency); + for (const operation of change.operations) { + if (operation.container.kind === "normal") + peers.add(operation.container.peer); + const content = operation.content; + if (content.type === "text-delete" || + content.type === "list-delete" || + content.type === "movable-list-delete") { + addId(content.startId); + } + else if (content.type === "tree-create" || content.type === "tree-move") { + addId(content.subject); + addId(content.parent); + } + else if (content.type === "tree-delete") { + addId(content.subject); + } + else if (content.type === "movable-list-move" || + content.type === "movable-list-set") { + peers.add(content.elementId.peer); + } + } + } + return [...peers].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); +} +function decodedOperationToJson(operation, keys, changePeer, peerMap) { + const operationId = { peer: changePeer, counter: operation.counter }; + const content = operation.content; + let json; + switch (content.type) { + case "map-insert": + json = { + type: "insert", + key: content.key, + value: changeLoroValueToJson(content.value, keys, operationId, peerMap), + }; + break; + case "map-delete": + json = { type: "delete", key: content.key }; + break; + case "text-insert": + json = { type: "insert", pos: content.position, text: content.value }; + break; + case "text-delete": + json = { + type: "delete", + pos: content.position, + len: Number(content.length), + start_id: formatJsonOpId(content.startId, peerMap), + }; + break; + case "text-mark": + json = { + type: "mark", + start: content.start, + end: content.end, + style_key: content.key, + style_value: changeLoroValueToJson(content.value, keys, operationId, peerMap), + info: content.info, + }; + break; + case "text-mark-end": + json = { type: "mark_end" }; + break; + case "list-insert": + case "movable-list-insert": + json = { + type: "insert", + pos: content.position, + value: content.values.map((value, index) => changeLoroValueToJson(value, keys, { peer: changePeer, counter: operation.counter + index }, peerMap)), + }; + break; + case "list-delete": + case "movable-list-delete": + json = { + type: "delete", + pos: content.position, + len: Number(content.length), + start_id: formatJsonOpId(content.startId, peerMap), + }; + break; + case "movable-list-move": + json = { + type: "move", + from: content.from, + to: content.to, + elem_id: formatJsonOpId({ peer: content.elementId.peer, counter: content.elementId.lamport }, peerMap), + }; + break; + case "movable-list-set": + json = { + type: "set", + elem_id: formatJsonOpId({ peer: content.elementId.peer, counter: content.elementId.lamport }, peerMap), + value: changeLoroValueToJson(content.value, keys, operationId, peerMap), + }; + break; + case "tree-create": + case "tree-move": + json = { + type: content.type === "tree-create" ? "create" : "move", + target: formatJsonTreeId(content.subject, peerMap), + parent: content.parent === undefined ? null : formatJsonTreeId(content.parent, peerMap), + fractional_index: bytesToHex(content.position).toUpperCase(), + }; + break; + case "tree-delete": + json = { type: "delete", target: formatJsonTreeId(content.subject, peerMap) }; + break; + case "future": { + const value = content.value; + if (value.type === "double" || value.type === "i64") { + json = { + type: "counter", + value: value.type === "double" ? value.value : Number(value.value), + prop: content.property, + }; + } + else if (value.type === "delta-int") { + json = { type: "counter", value: value.value, prop: content.property }; + } + else { + json = { + type: "unknown", + prop: content.property, + value: changeValueToJsonUnknown(value), + }; + } + break; + } + } + return { + container: formatJsonContainerId(operation.container, peerMap), + content: json, + counter: operation.counter, + }; +} +function changeLoroValueToJson(value, keys, operationId, peerMap) { + switch (value.type) { + case "null": + return null; + case "bool": + case "double": + case "string": + return value.value; + case "i64": + return Number(value.value); + case "binary": + return value.value.slice(); + case "list": + return value.value.map((item, index) => changeLoroValueToJson(item, keys, { peer: operationId.peer, counter: operationId.counter + index }, peerMap)); + case "map": + return Object.fromEntries(value.value.map(([keyIndex, item]) => { + const key = keys[Number(keyIndex)]; + if (key === undefined) + throw new RangeError("JSON value key is out of range"); + return [key, changeLoroValueToJson(item, keys, operationId, peerMap)]; + })); + case "container-type": { + const child = { + kind: "normal", + ...operationId, + containerType: containerTypeFromRawByte(value.value), + }; + return `🦜:${formatJsonContainerId(child, peerMap)}`; + } + } +} +function changeValueToJsonUnknown(value) { + if (typeof value !== "object" || value === null) + return value; + if ("value" in value) { + const inner = value.value; + return typeof inner === "bigint" ? Number(inner) : inner; + } + if ("data" in value) + return value.data.slice(); + return value; +} +function formatJsonOpId(id, peerMap) { + return `${id.counter}@${(peerMap?.get(id.peer) ?? id.peer).toString()}`; +} +function formatJsonTreeId(id, peerMap) { + return formatJsonOpId(id, peerMap); +} +function formatJsonContainerId(id, peerMap) { + return formatContainerId(id.kind === "normal" ? { ...id, peer: peerMap?.get(id.peer) ?? id.peer } : id); +} +function jsonSchemaToHistoryRecords(schema) { + if (!Array.isArray(schema.changes)) + throw new TypeError("JSON changes must be an array"); + const peers = Array.isArray(schema.peers) ? schema.peers.map(parsePeerId) : undefined; + const resolvePeer = (peer) => { + if (peers === undefined) + return peer; + const resolved = peers[Number(peer)]; + if (resolved === undefined) + throw new RangeError(`JSON peer index ${peer} is missing`); + return resolved; + }; + const parseJsonId = (value) => { + const parsed = parseTreeId(value); + return { peer: resolvePeer(parsed.peer), counter: parsed.counter }; + }; + const parseJsonContainer = (value) => { + const parsed = parseContainerId(value); + return parsed.kind === "normal" + ? { ...parsed, peer: resolvePeer(parsed.peer) } + : parsed; + }; + return schema.changes.map((change) => { + const id = parseJsonId(change.id); + const keys = []; + const keyIndices = new Map(); + const registerJsonKey = (key) => { + const existing = keyIndices.get(key); + if (existing !== undefined) + return existing; + const index = keys.length; + keys.push(key); + keyIndices.set(key, index); + return index; + }; + const operations = change.ops.map((operation) => jsonOperationToDecoded(operation, id.peer, resolvePeer, parseJsonContainer, registerJsonKey)); + let expectedCounter = id.counter; + for (const operation of operations) { + if (operation.counter !== expectedCounter) { + throw new RangeError(`JSON operation counter ${operation.counter} does not follow ${expectedCounter}`); + } + expectedCounter += operation.length; + } + return { + change: { + id, + timestamp: BigInt(Math.trunc(change.timestamp)), + dependencies: change.deps.map(parseJsonId), + lamport: change.lamport, + message: change.msg ?? undefined, + operations, + }, + keys, + keyIndices: undefined, + }; + }); +} +function jsonOperationToDecoded(operation, changePeer, resolvePeer, parseJsonContainer, registerKey) { + const container = parseJsonContainer(operation.container); + const containerType = codecTypeToPublic(container.containerType); + const content = operation.content; + const operationId = { peer: changePeer, counter: operation.counter }; + const parseId = (value) => { + if (typeof value !== "string") + throw new TypeError("JSON operation ID must be a string"); + const parsed = parseTreeId(value); + return { peer: resolvePeer(parsed.peer), counter: parsed.counter }; + }; + const parseParent = (value) => value === null || value === undefined ? undefined : parseId(value); + const encodeValue = (value, id = operationId) => jsonValueToChangeLoroValue(value, id, resolvePeer, registerKey); + if (containerType === "Map") { + if (content.type === "delete") { + if (typeof content.key !== "string") + throw new TypeError("map delete key is missing"); + return { + container, + counter: operation.counter, + length: 1, + content: { type: "map-delete", key: content.key }, + }; + } + if (content.type !== "insert" || typeof content.key !== "string") { + throw new TypeError("invalid map JSON operation"); + } + return { + container, + counter: operation.counter, + length: 1, + content: { + type: "map-insert", + key: content.key, + value: encodeValue(content.value), + }, + }; + } + if (containerType === "Text") { + if (content.type === "insert" && typeof content.text === "string") { + return { + container, + counter: operation.counter, + length: Array.from(content.text).length, + content: { + type: "text-insert", + position: requireJsonInteger(content.pos, "text insert position"), + value: content.text, + }, + }; + } + if (content.type === "delete") { + const length = requireJsonInteger(content.len, "text delete length"); + return { + container, + counter: operation.counter, + length: Math.abs(length), + content: { + type: "text-delete", + position: requireJsonInteger(content.pos, "text delete position"), + length: BigInt(length), + startId: parseId(content.start_id), + }, + }; + } + if (content.type === "mark") { + if (typeof content.style_key !== "string") + throw new TypeError("text mark key is missing"); + return { + container, + counter: operation.counter, + length: 1, + content: { + type: "text-mark", + start: requireJsonInteger(content.start, "text mark start"), + end: requireJsonInteger(content.end, "text mark end"), + key: content.style_key, + value: encodeValue(content.style_value), + info: requireJsonInteger(content.info, "text mark info"), + }, + }; + } + if (content.type === "mark_end") { + return { + container, + counter: operation.counter, + length: 1, + content: { type: "text-mark-end" }, + }; + } + throw new TypeError("invalid text JSON operation"); + } + if (containerType === "List" || containerType === "MovableList") { + const movable = containerType === "MovableList"; + if (content.type === "insert") { + if (!Array.isArray(content.value)) + throw new TypeError("list insert value must be an array"); + const values = content.value.map((value, index) => encodeValue(value, { peer: changePeer, counter: operation.counter + index })); + return { + container, + counter: operation.counter, + length: values.length, + content: { + type: movable ? "movable-list-insert" : "list-insert", + position: requireJsonInteger(content.pos, "list insert position"), + values, + }, + }; + } + if (content.type === "delete") { + const length = requireJsonInteger(content.len, "list delete length"); + return { + container, + counter: operation.counter, + length: Math.abs(length), + content: { + type: movable ? "movable-list-delete" : "list-delete", + position: requireJsonInteger(content.pos, "list delete position"), + length: BigInt(length), + startId: parseId(content.start_id), + }, + }; + } + if (movable && content.type === "move") { + const elementId = parseId(content.elem_id); + return { + container, + counter: operation.counter, + length: 1, + content: { + type: "movable-list-move", + from: requireJsonInteger(content.from, "movable-list source"), + to: requireJsonInteger(content.to, "movable-list destination"), + elementId: { peer: elementId.peer, lamport: elementId.counter }, + }, + }; + } + if (movable && content.type === "set") { + const elementId = parseId(content.elem_id); + return { + container, + counter: operation.counter, + length: 1, + content: { + type: "movable-list-set", + elementId: { peer: elementId.peer, lamport: elementId.counter }, + value: encodeValue(content.value), + }, + }; + } + throw new TypeError("invalid list JSON operation"); + } + if (containerType === "Tree") { + if (content.type === "delete") { + return { + container, + counter: operation.counter, + length: 1, + content: { type: "tree-delete", subject: parseId(content.target) }, + }; + } + if (content.type !== "create" && content.type !== "move") { + throw new TypeError("invalid tree JSON operation"); + } + const position = typeof content.fractional_index === "string" + ? hexToBytes(content.fractional_index) + : Uint8Array.of(0x80); + return { + container, + counter: operation.counter, + length: 1, + content: { + type: content.type === "create" ? "tree-create" : "tree-move", + subject: parseId(content.target), + parent: parseParent(content.parent), + position, + }, + }; + } + if (containerType === "Counter") { + const value = typeof content === "number" + ? content + : typeof content.value === "number" + ? content.value + : Number.NaN; + if (!Number.isFinite(value)) + throw new TypeError("counter JSON value must be finite"); + return { + container, + counter: operation.counter, + length: 1, + content: { + type: "future", + property: typeof content === "object" && content !== null && "prop" in content + ? requireJsonInteger(content.prop, "counter property") + : 0, + value: { type: "double", value }, + }, + }; + } + throw new TypeError("JSON updates do not support this container type"); +} +function jsonValueToChangeLoroValue(value, operationId, resolvePeer, registerKey) { + if (value === undefined || value === null) + return { type: "null" }; + if (typeof value === "boolean") + return { type: "bool", value }; + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new TypeError("JSON update numbers must be finite"); + return Number.isSafeInteger(value) + ? { type: "i64", value: BigInt(value) } + : { type: "double", value }; + } + if (typeof value === "bigint") + return { type: "i64", value }; + if (typeof value === "string") { + if (value.startsWith("🦜:")) { + const rawId = value.slice("🦜:".length); + if (isContainerId(rawId)) { + const child = parseContainerId(rawId); + if (child.kind !== "normal") { + throw new TypeError("JSON child references must use normal container IDs"); + } + const peer = resolvePeer(child.peer); + if (peer !== operationId.peer || child.counter !== operationId.counter) { + throw new RangeError("JSON child container ID does not match its operation ID"); + } + return { + type: "container-type", + value: containerTypeToRawByte(child.containerType), + }; + } + } + return { type: "string", value }; + } + if (value instanceof Uint8Array) + return { type: "binary", value: value.slice() }; + if (Array.isArray(value)) { + return { + type: "list", + value: value.map((item, index) => jsonValueToChangeLoroValue(item, { peer: operationId.peer, counter: operationId.counter + index }, resolvePeer, registerKey)), + }; + } + if (typeof value === "object") { + return { + type: "map", + value: Object.entries(value).map(([key, item]) => [ + BigInt(registerKey(key)), + jsonValueToChangeLoroValue(item, operationId, resolvePeer, registerKey), + ]), + }; + } + throw new TypeError(`unsupported JSON update value type: ${typeof value}`); +} +function requireJsonInteger(value, label) { + if (!Number.isSafeInteger(value)) + throw new TypeError(`${label} must be an integer`); + return value; +} +function unicodeScalarLength(value) { + let length = 0; + for (let index = 0; index < value.length; index += 1) { + const first = value.charCodeAt(index); + if (first >= 0xd800 && + first <= 0xdbff && + index + 1 < value.length && + value.charCodeAt(index + 1) >= 0xdc00 && + value.charCodeAt(index + 1) <= 0xdfff) { + index += 1; + } + length += 1; + } + return length; +} +function compareWriter(left, right) { + return (left.lamport - right.lamport || + (left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0)); +} +function operationWriter(change, operation) { + return { + peer: change.id.peer, + lamport: change.lamport + operation.counter - change.id.counter, + }; +} +function captureBlueprint(container) { + if (container instanceof LoroText) + return { kind: "Text", value: container.toDelta() }; + if (container instanceof LoroCounter) + return { kind: "Counter", value: container.value }; + if (container instanceof LoroList) { + return { + kind: container.kind(), + value: container._visibleElements().map((element) => element.value), + }; + } + return { + kind: container.kind(), + value: container instanceof LoroMap ? container.entries() : container.toJSON(), + }; +} +function restoreBlueprint(container, blueprint) { + if (container instanceof LoroMap) { + for (const [key, value] of blueprint.value) { + if (isContainer(value)) + container.setContainer(key, value); + else + container.set(key, value); + } + } + else if (container instanceof LoroText) { + container.applyDelta(blueprint.value); + } + else if (container instanceof LoroCounter) { + if (blueprint.value !== 0) + container.increment(blueprint.value); + } + else if (container instanceof LoroList) { + for (const value of blueprint.value) { + if (isContainer(value)) + container.pushContainer(value); + else + container.push(value); + } + } +} +function containerDepth(container) { + let depth = 1; + let parent = container.parent(); + while (parent !== undefined) { + depth += 1; + parent = parent.parent(); + } + return depth; +} +function assertTextStyleExpand(value) { + if (value !== "before" && value !== "after" && value !== "none" && value !== "both") { + throw new TypeError(`invalid text style expand mode: ${value}`); + } +} +function textStyleInfoByte(expand, deleting) { + const effective = deleting + ? expand === "none" + ? "both" + : expand === "both" + ? "none" + : expand + : expand; + return (0x80 | + (effective === "before" || effective === "both" ? 0x02 : 0) | + (effective === "after" || effective === "both" ? 0x04 : 0)); +} +function generatePeerId() { + const cryptoObject = globalThis.crypto; + if (cryptoObject !== undefined) { + const bytes = new Uint8Array(8); + cryptoObject.getRandomValues(bytes); + let value = 0n; + for (const byte of bytes) + value = (value << 8n) | BigInt(byte); + if (value !== 0n) + return value; + } + return fallbackPeer++; +} +function importStatus(records, pendingRecords) { + // Group by the bigint peer first so the string form is built once per peer + // rather than once per change. + const spansByPeer = new Map(); + for (const { change } of records) { + const start = change.id.counter; + const end = start + changeLength(change); + const current = spansByPeer.get(change.id.peer); + if (current === undefined) { + spansByPeer.set(change.id.peer, { start, end }); + } + else { + current.start = Math.min(start, current.start); + current.end = Math.max(end, current.end); + } + } + const success = new Map(); + for (const [peer, span] of spansByPeer) { + success.set(peerIdToString(peer), span); + } + const pending = historySpans(pendingRecords); + return { success, pending: pending.size === 0 ? null : pending }; +} +function importStatusBetweenVersions(before, retainedStart, end) { + const success = new Map(); + for (const { peer, counter } of end._codecEntriesUnsorted()) { + const start = Math.max(before.get(peer) ?? 0, retainedStart.get(peer) ?? 0); + if (counter > start) { + success.set(peerIdToString(peer), { start, end: counter }); + } + } + return { success, pending: null }; +} +function historySpans(records) { + const spansByPeer = new Map(); + for (const { change } of records) { + const start = change.id.counter; + const end = start + changeLength(change); + const current = spansByPeer.get(change.id.peer); + if (current === undefined) { + spansByPeer.set(change.id.peer, { start, end }); + } + else { + current.start = Math.min(start, current.start); + current.end = Math.max(end, current.end); + } + } + const spans = new Map(); + for (const [peer, span] of spansByPeer) { + spans.set(peerIdToString(peer), span); + } + return spans; +} +function isEmptyJson(value) { + return (value === "" || + (Array.isArray(value) && value.length === 0) || + (typeof value === "object" && value !== null && Object.keys(value).length === 0) || + value === 0); +} +function containerEventValue(container) { + if (container instanceof LoroMap) + return new Map(container.entries()); + if (container instanceof LoroText) { + return { + text: container.toString(), + delta: container.toDelta(), + }; + } + if (container instanceof LoroTree) { + return container.getNodes().map((node) => ({ + id: node.id, + parent: node.parent()?.id, + index: node.index(), + fractionalIndex: node.fractionalIndex() ?? "", + })); + } + if (container instanceof LoroCounter) + return container.value; + return container.toArray(); +} +function containerDiff(container, beforeValue, mapOperationKeys = { from: undefined, to: undefined }) { + if (container instanceof LoroMap) { + const before = beforeValue instanceof Map ? beforeValue : new Map(); + const after = new Map(container.entries()); + const updated = {}; + const keys = new Set([ + ...before.keys(), + ...after.keys(), + ...(mapOperationKeys.from ?? []), + ...(mapOperationKeys.to ?? []), + ]); + for (const key of keys) { + const previous = before.get(key); + const next = after.get(key); + const tombstonePresenceChanged = !before.has(key) && + !after.has(key) && + mapOperationKeys.from?.has(key) !== mapOperationKeys.to?.has(key); + if (tombstonePresenceChanged || + !eventValuesEqual(previous, next) || + before.has(key) !== after.has(key)) { + updated[key] = next; + } + } + return { type: "map", updated }; + } + if (container instanceof LoroText) { + const before = isTextEventValue(beforeValue) + ? beforeValue + : ({ text: "", delta: [] }); + const after = containerEventValue(container); + if (eventValuesEqual(before.delta, after.delta)) + return { type: "text", diff: [] }; + const styled = [...before.delta, ...after.delta].some((item) => "attributes" in item && item.attributes !== undefined); + if (styled) { + return { + type: "text", + diff: [ + ...(before.text.length === 0 ? [] : [{ delete: before.text.length }]), + ...after.delta, + ], + }; + } + return { type: "text", diff: stringDelta(before.text, after.text) }; + } + if (container instanceof LoroTree) { + return { + type: "tree", + diff: treeDelta(Array.isArray(beforeValue) ? beforeValue : [], containerEventValue(container)), + }; + } + if (container instanceof LoroCounter) { + return { + type: "counter", + increment: container.value - (typeof beforeValue === "number" ? beforeValue : 0), + }; + } + return { + type: "list", + diff: listDelta(Array.isArray(beforeValue) ? beforeValue : [], container.toArray()), + }; +} +function diffForJson(diff) { + if (diff.type === "map") { + return { + type: "map", + updated: Object.fromEntries(Object.entries(diff.updated).map(([key, value]) => [key, jsonDiffValue(value)])), + }; + } + if (diff.type === "list") { + return { + type: "list", + diff: diff.diff.map((operation) => "insert" in operation + ? { ...operation, insert: operation.insert.map(jsonDiffValue) } + : operation), + }; + } + return diff; +} +function jsonDiffValue(value) { + if (isContainer(value)) + return `🦜:${value.id}`; + if (value instanceof Uint8Array || value === null) + return value; + if (Array.isArray(value)) + return value.map(jsonDiffValue); + if (typeof value === "object" && value !== null) { + return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, jsonDiffValue(child)])); + } + return value; +} +// True when a UTF-16 boundary at `at` would split a surrogate pair. +function splitsSurrogatePair(value, at) { + if (at <= 0 || at >= value.length) + return false; + const before = value.charCodeAt(at - 1); + const after = value.charCodeAt(at); + return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff; +} +function stringDelta(before, after) { + // Scan the common prefix/suffix directly in UTF-16 code units and only + // allocate the inserted substring. Boundaries are backed off to code-point + // edges so the delta never splits a surrogate pair. + let prefix = 0; + const prefixLimit = Math.min(before.length, after.length); + while (prefix < prefixLimit && before.charCodeAt(prefix) === after.charCodeAt(prefix)) { + prefix += 1; + } + if (splitsSurrogatePair(before, prefix) || splitsSurrogatePair(after, prefix)) { + prefix -= 1; + } + let suffix = 0; + const suffixLimit = Math.min(before.length - prefix, after.length - prefix); + while (suffix < suffixLimit && + before.charCodeAt(before.length - suffix - 1) === + after.charCodeAt(after.length - suffix - 1)) { + suffix += 1; + } + if (splitsSurrogatePair(before, before.length - suffix) || + splitsSurrogatePair(after, after.length - suffix)) { + suffix -= 1; + } + const diff = []; + if (prefix > 0) + diff.push({ retain: prefix }); + const deleted = before.length - prefix - suffix; + if (deleted > 0) + diff.push({ delete: deleted }); + const inserted = after.slice(prefix, after.length - suffix); + if (inserted.length > 0) + diff.push({ insert: inserted }); + return diff; +} +function listDelta(before, after) { + let prefix = 0; + while (prefix < before.length && + prefix < after.length && + eventValuesEqual(before[prefix], after[prefix])) { + prefix += 1; + } + let suffix = 0; + while (suffix < before.length - prefix && + suffix < after.length - prefix && + eventValuesEqual(before[before.length - suffix - 1], after[after.length - suffix - 1])) { + suffix += 1; + } + const diff = []; + if (prefix > 0) + diff.push({ retain: prefix }); + const deleted = before.length - prefix - suffix; + if (deleted > 0) + diff.push({ delete: deleted }); + const inserted = after.slice(prefix, after.length - suffix); + if (inserted.length > 0) + diff.push({ insert: [...inserted] }); + return diff; +} +function treeDelta(before, after) { + const beforeById = new Map(before.map((node) => [node.id, node])); + const afterById = new Map(after.map((node) => [node.id, node])); + const diff = []; + for (const node of before) { + if (!afterById.has(node.id)) { + diff.push({ + target: node.id, + action: "delete", + oldParent: node.parent, + oldIndex: node.index, + }); + } + } + for (const node of after) { + const previous = beforeById.get(node.id); + if (previous === undefined) { + diff.push({ + target: node.id, + action: "create", + parent: node.parent, + index: node.index, + fractionalIndex: node.fractionalIndex, + }); + } + else if (previous.parent !== node.parent || + previous.index !== node.index || + previous.fractionalIndex !== node.fractionalIndex) { + diff.push({ + target: node.id, + action: "move", + parent: node.parent, + index: node.index, + fractionalIndex: node.fractionalIndex, + oldParent: previous.parent, + oldIndex: previous.index, + }); + } + } + return diff; +} +function eventValuesEqual(left, right) { + if (left === right) + return true; + // undefined compares equal to null, matching how the runtime encodes both + // as the null Loro value. Stored values never contain undefined, so this + // only normalizes user-provided input. + if (left === undefined) + left = null; + if (right === undefined) + right = null; + if (left === right) + return true; + if (left instanceof Uint8Array && right instanceof Uint8Array) { + return bytesEqual(left, right); + } + if (left instanceof Uint8Array || right instanceof Uint8Array) + return false; + if (isContainer(left) && isContainer(right)) + return left.id === right.id; + if (isContainer(left) || isContainer(right)) + return false; + if (Array.isArray(left) && Array.isArray(right)) { + if (left.length !== right.length) + return false; + for (let index = 0; index < left.length; index += 1) { + if (!eventValuesEqual(left[index], right[index])) + return false; + } + return true; + } + if (Array.isArray(left) || Array.isArray(right)) + return false; + if (left instanceof Map && right instanceof Map) { + if (left.size !== right.size) + return false; + for (const [key, value] of left) { + if (!right.has(key) || !eventValuesEqual(value, right.get(key))) + return false; + } + return true; + } + if (left instanceof Map || right instanceof Map) + return false; + if (typeof left === "object" && + left !== null && + typeof right === "object" && + right !== null) { + const leftRecord = left; + const rightRecord = right; + let leftCount = 0; + for (const key in leftRecord) { + leftCount += 1; + if (!(key in rightRecord) || !eventValuesEqual(leftRecord[key], rightRecord[key])) { + return false; + } + } + let rightCount = 0; + for (const _key in rightRecord) + rightCount += 1; + return leftCount === rightCount; + } + return false; +} +function subtractSequenceIdRuns(runs, removedRuns) { + if (removedRuns.length === 0) + return [...runs]; + const removalsByPeer = new Map(); + for (const run of removedRuns) { + let removals = removalsByPeer.get(run.start.peer); + if (removals === undefined) { + removals = []; + removalsByPeer.set(run.start.peer, removals); + } + removals.push({ + start: run.start.counter, + end: run.start.counter + run.length, + }); + } + for (const removals of removalsByPeer.values()) { + removals.sort((left, right) => left.start - right.start); + let write = 0; + for (const removal of removals) { + const previous = write > 0 ? removals[write - 1] : undefined; + if (previous !== undefined && removal.start <= previous.end) { + previous.end = Math.max(previous.end, removal.end); + } + else { + removals[write++] = removal; + } + } + removals.length = write; + } + const retained = []; + for (const run of runs) { + const removals = removalsByPeer.get(run.start.peer); + if (removals === undefined) { + retained.push(run); + continue; + } + const end = run.start.counter + run.length; + let low = 0; + let high = removals.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (removals[middle].end <= run.start.counter) + low = middle + 1; + else + high = middle; + } + let cursor = run.start.counter; + for (let index = low; index < removals.length; index += 1) { + const removal = removals[index]; + if (removal.start >= end) + break; + if (cursor < removal.start) { + retained.push({ + start: { peer: run.start.peer, counter: cursor }, + length: Math.min(removal.start, end) - cursor, + }); + } + cursor = Math.max(cursor, removal.end); + if (cursor >= end) + break; + } + if (cursor < end) { + retained.push({ + start: { peer: run.start.peer, counter: cursor }, + length: end - cursor, + }); + } + } + return retained; +} +function isTextEventValue(value) { + return (typeof value === "object" && + value !== null && + typeof value.text === "string" && + Array.isArray(value.delta)); +} +function isEmptyContainerDiff(diff) { + if (diff.type === "map") + return Object.keys(diff.updated).length === 0; + if (diff.type === "counter") + return diff.increment === 0; + return diff.diff.length === 0; +} +function recoverParentBinding(child, parent) { + if (parent instanceof LoroMap) { + for (const [key, record] of parent._entries) { + if (record.value !== child) + continue; + const binding = { kind: "map", key }; + child._setParentBinding(parent, binding); + return binding; + } + } + else if (parent instanceof LoroList) { + for (const element of parent._sequence.all()) { + if (element.value !== child) + continue; + const binding = { kind: "sequence", element }; + child._setParentBinding(parent, binding); + return binding; + } + } + else if (parent instanceof LoroTree) { + for (const record of parent._nodes.values()) { + if (record.data !== child) + continue; + const binding = { kind: "tree", record }; + child._setParentBinding(parent, binding); + return binding; + } + } + return undefined; +} +function containerPath(container) { + const path = []; + let current = container; + while (current !== undefined) { + const id = current._codecId; + if (id?.kind === "root" && !isMergeableContainerId(id)) { + path.unshift(id.name); + break; + } + const parent = current.parent(); + const binding = current._parentLink?.binding ?? + (parent === undefined ? undefined : recoverParentBinding(current, parent)); + if (parent instanceof LoroMap) { + if (binding?.kind === "map") { + path.unshift(binding.key); + } + } + else if (parent instanceof LoroList) { + const index = binding?.kind === "sequence" + ? parent._sequence.visibleIndexOf(binding.element) + : undefined; + if (index !== undefined && index >= 0) + path.unshift(index); + } + else if (parent instanceof LoroTree) { + if (binding?.kind === "tree") + path.unshift(formatTreeId(binding.record.id)); + else if (id?.kind === "normal") + path.unshift(formatTreeId(id)); + } + current = parent; + } + return path; +} +function parseOptionalPathIndex(value) { + if (!/^(0|[1-9]\d*)$/u.test(value)) + return undefined; + const index = Number(value); + return Number.isSafeInteger(index) ? index : undefined; +} +function parsePathIndex(value) { + return parseOptionalPathIndex(value) ?? -1; +} +function treeNodeAtPath(tree, _parent, part) { + if (part.includes("@")) + return tree.getNodeByID(part); + const index = parseOptionalPathIndex(part); + return index === undefined ? undefined : tree._nodeAt(undefined, index); +} diff --git a/loro-js/src/runtime/ephemeral.js b/loro-js/src/runtime/ephemeral.js new file mode 100644 index 000000000..ff610c09e --- /dev/null +++ b/loro-js/src/runtime/ephemeral.js @@ -0,0 +1,560 @@ +import { PostcardReader, PostcardWriter } from "../codec/postcard"; +import { readPostcardValue, writePostcardValue } from "../codec/value"; +import { formatContainerId, isContainerId, parseContainerId, parsePeerId } from "./ids"; +/** The low-level awareness class exported by `loro-crdt`. */ +export class AwarenessWasm { + #peer; + #timeout; + #states = new Map(); + constructor(peer, timeout) { + this.#peer = parsePeerId(peer); + this.#timeout = assertTimeout(timeout); + } + free() { } + encode(peers) { + const now = Date.now(); + const states = []; + for (const peerInput of peers) { + const peer = parsePeerId(peerInput); + const state = this.#states.get(peer); + if (state === undefined || now - state.timestamp > this.#timeout) + continue; + states.push({ peer, counter: state.counter, value: state.value }); + } + return encodeAwarenessStates(states); + } + encodeAll() { + return this.encode(this.peers()); + } + apply(bytes) { + let decoded; + try { + decoded = decodeAwarenessStates(bytes); + } + catch (error) { + throw new Error(`Failed to decode awareness data: ${errorMessage(error)}`); + } + const now = Date.now(); + const updated = []; + const added = []; + for (const state of decoded) { + const current = this.#states.get(state.peer); + if (state.peer === this.#peer || + (current !== undefined && current.counter >= state.counter)) { + continue; + } + this.#states.set(state.peer, { + value: state.value, + counter: state.counter, + timestamp: now, + }); + (current === undefined ? added : updated).push(peerString(state.peer)); + } + return { updated, added }; + } + setLocalState(value) { + const current = this.#states.get(this.#peer); + this.#states.set(this.#peer, { + value: valueToEncoded(value), + counter: (current?.counter ?? 0) + 1, + timestamp: Date.now(), + }); + } + peer() { + return peerString(this.#peer); + } + getAllStates() { + const output = {}; + for (const [peer, state] of this.#states) { + output[peerString(peer)] = encodedToValue(state.value); + } + return output; + } + getState(peer) { + const state = this.#states.get(parsePeerId(peer)); + return state === undefined ? undefined : encodedToValue(state.value); + } + getTimestamp(peer) { + return this.#states.get(parsePeerId(peer))?.timestamp; + } + removeOutdated() { + const now = Date.now(); + const removed = []; + for (const peer of this.orderedPeers()) { + const state = this.#states.get(peer); + if (now - state.timestamp <= this.#timeout) + continue; + this.#states.delete(peer); + removed.push(peerString(peer)); + } + return removed; + } + length() { + return this.#states.size; + } + isEmpty() { + return this.#states.size === 0; + } + peers() { + return this.orderedPeers().map(peerString); + } + orderedPeers() { + const peers = []; + if (this.#states.has(this.#peer)) + peers.push(this.#peer); + for (const peer of this.#states.keys()) { + if (peer !== this.#peer) + peers.push(peer); + } + return peers; + } +} +/** @deprecated Use `EphemeralStore` for new code. */ +export class Awareness { + inner; + #peer; + #timeout; + #listeners = new Set(); + #timer; + constructor(peer, timeout = 30000) { + this.inner = new AwarenessWasm(peer, timeout); + this.#peer = this.inner.peer(); + this.#timeout = timeout; + } + apply(bytes, origin = "remote") { + const { updated, added } = this.inner.apply(bytes); + for (const listener of this.#listeners) { + listener({ updated, added, removed: [] }, origin); + } + this.startTimerIfNotEmpty(); + } + setLocalState(state) { + const wasEmpty = this.inner.getState(this.#peer) === undefined; + this.inner.setLocalState(state); + const peer = this.inner.peer(); + for (const listener of this.#listeners) { + listener(wasEmpty + ? { updated: [], added: [peer], removed: [] } + : { updated: [peer], added: [], removed: [] }, "local"); + } + this.startTimerIfNotEmpty(); + } + getLocalState() { + return this.inner.getState(this.#peer); + } + getAllStates() { + return this.inner.getAllStates(); + } + encode(peers) { + return this.inner.encode(peers); + } + encodeAll() { + return this.inner.encodeAll(); + } + addListener(listener) { + this.#listeners.add(listener); + } + removeListener(listener) { + this.#listeners.delete(listener); + } + peers() { + return this.inner.peers(); + } + destroy() { + if (this.#timer !== undefined) + clearInterval(this.#timer); + this.#timer = undefined; + this.#listeners.clear(); + } + startTimerIfNotEmpty() { + if (this.inner.isEmpty() || this.#timer !== undefined) + return; + this.#timer = setInterval(() => { + const removed = this.inner.removeOutdated(); + if (removed.length > 0) { + for (const listener of this.#listeners) { + listener({ updated: [], added: [], removed }, "timeout"); + } + } + if (!this.inner.isEmpty()) + return; + clearInterval(this.#timer); + this.#timer = undefined; + }, this.#timeout / 2); + } +} +/** The low-level ephemeral key-value store exported by `loro-crdt`. */ +export class EphemeralStoreWasm { + #timeout; + #states = new Map(); + #listeners = new Set(); + #localListeners = new Set(); + #pendingEvents = []; + #pendingLocalUpdates = []; + // Adds made from inside a dispatch are queued and flushed after that + // event's dispatch, preserving the former per-event snapshot semantics + // without copying the listener set per emission. Removals apply to the + // live set immediately; Set iteration skips not-yet-visited removals, + // which matches the old `has` guard. + #queuedListeners = []; + #queuedLocalListeners = []; + #deferringListeners = false; + #deferringLocalListeners = false; + #emittingEvents = false; + #emittingLocalUpdates = false; + constructor(timeout) { + this.#timeout = assertTimeout(timeout); + } + free() { } + set(key, value) { + this.setLocal(key, valueToEncoded(value)); + } + delete(key) { + this.setLocal(key, undefined); + } + get(key) { + const value = this.#states.get(key)?.value; + return value === undefined ? undefined : encodedToValue(value); + } + getAllStates() { + const output = {}; + for (const [key, state] of this.#states) { + if (state.value !== undefined) + output[key] = encodedToValue(state.value); + } + return output; + } + subscribeLocalUpdates(listener) { + if (this.#deferringLocalListeners) + this.#queuedLocalListeners.push(listener); + else + this.#localListeners.add(listener); + return () => { + this.#localListeners.delete(listener); + const queued = this.#queuedLocalListeners.indexOf(listener); + if (queued !== -1) + this.#queuedLocalListeners.splice(queued, 1); + }; + } + subscribe(listener) { + if (this.#deferringListeners) + this.#queuedListeners.push(listener); + else + this.#listeners.add(listener); + return () => { + this.#listeners.delete(listener); + const queued = this.#queuedListeners.indexOf(listener); + if (queued !== -1) + this.#queuedListeners.splice(queued, 1); + }; + } + encode(key) { + const state = this.#states.get(key); + if (state === undefined) + return encodeEphemeralStates([]); + if (Date.now() - state.timestamp > this.#timeout) + return new Uint8Array(); + return encodeEphemeralStates([{ key, ...state }]); + } + encodeAll() { + const now = Date.now(); + const states = []; + for (const [key, state] of this.#states) { + if (now - state.timestamp <= this.#timeout) + states.push({ key, ...state }); + } + return encodeEphemeralStates(states); + } + apply(bytes) { + let decoded; + try { + decoded = decodeEphemeralStates(bytes); + } + catch (error) { + throw new Error(`Failed to decode data: ${errorMessage(error)}`); + } + const added = []; + const updated = []; + const removed = []; + const now = Date.now(); + for (const state of decoded) { + if (now - state.timestamp > this.#timeout) + continue; + const old = this.#states.get(state.key); + if (old !== undefined && old.timestamp >= state.timestamp) + continue; + this.#states.set(state.key, { value: state.value, timestamp: state.timestamp }); + if (old !== undefined && state.value !== undefined) + updated.push(state.key); + else if (old === undefined && state.value !== undefined) + added.push(state.key); + else if (old !== undefined && state.value === undefined) + removed.push(state.key); + } + this.emit({ by: "import", added, updated, removed }); + } + removeOutdated() { + const now = Date.now(); + const removed = []; + for (const [key, state] of this.#states) { + if (now - state.timestamp <= this.#timeout) + continue; + this.#states.delete(key); + if (state.value !== undefined) + removed.push(key); + } + this.emit({ by: "timeout", added: [], updated: [], removed }); + } + isEmpty() { + return this.#states.size === 0; + } + keys() { + const output = []; + for (const [key, state] of this.#states) { + if (state.value !== undefined) + output.push(key); + } + return output; + } + setLocal(key, value) { + const old = this.#states.get(key); + this.#states.set(key, { value, timestamp: Date.now() }); + const bytes = this.encode(key); + this.emitLocalUpdate(bytes); + const isDelete = value === undefined; + this.emit({ + by: "local", + added: old === undefined && !isDelete ? [key] : [], + updated: old !== undefined && !isDelete ? [key] : [], + removed: old !== undefined && isDelete ? [key] : [], + }); + } + emit(event) { + if (this.#emittingEvents) { + this.#pendingEvents.push(event); + return; + } + this.#emittingEvents = true; + const pending = [event]; + try { + while (pending.length > 0) { + const current = pending.pop(); + this.#deferringListeners = true; + try { + for (const listener of this.#listeners) + listener(current); + } + finally { + this.#deferringListeners = false; + for (const listener of this.#queuedListeners) + this.#listeners.add(listener); + this.#queuedListeners.length = 0; + } + pending.push(...this.#pendingEvents.splice(0)); + } + } + finally { + this.#emittingEvents = false; + this.#pendingEvents.length = 0; + } + } + emitLocalUpdate(bytes) { + if (this.#emittingLocalUpdates) { + this.#pendingLocalUpdates.push(bytes); + return; + } + this.#emittingLocalUpdates = true; + const pending = [bytes]; + try { + while (pending.length > 0) { + const current = pending.pop(); + this.#deferringLocalListeners = true; + try { + for (const listener of this.#localListeners) + listener(current); + } + finally { + this.#deferringLocalListeners = false; + for (const listener of this.#queuedLocalListeners) { + this.#localListeners.add(listener); + } + this.#queuedLocalListeners.length = 0; + } + pending.push(...this.#pendingLocalUpdates.splice(0)); + } + } + finally { + this.#emittingLocalUpdates = false; + this.#pendingLocalUpdates.length = 0; + } + } +} +/** A typed, automatically expiring wrapper around `EphemeralStoreWasm`. */ +export class EphemeralStore { + inner; + #timeout; + #timer; + constructor(timeout = 30000) { + this.inner = new EphemeralStoreWasm(timeout); + this.#timeout = timeout; + } + apply(bytes) { + this.inner.apply(bytes); + this.startTimerIfNotEmpty(); + } + set(key, value) { + this.inner.set(key, value); + this.startTimerIfNotEmpty(); + } + delete(key) { + this.inner.delete(key); + } + get(key) { + return this.inner.get(key); + } + getAllStates() { + return this.inner.getAllStates(); + } + encode(key) { + return this.inner.encode(key); + } + encodeAll() { + return this.inner.encodeAll(); + } + keys() { + return this.inner.keys(); + } + destroy() { + if (this.#timer !== undefined) + clearInterval(this.#timer); + this.#timer = undefined; + } + subscribe(listener) { + return this.inner.subscribe(listener); + } + subscribeLocalUpdates(listener) { + return this.inner.subscribeLocalUpdates(listener); + } + startTimerIfNotEmpty() { + if (this.inner.isEmpty() || this.#timer !== undefined) + return; + this.#timer = setInterval(() => { + this.inner.removeOutdated(); + if (!this.inner.isEmpty()) + return; + clearInterval(this.#timer); + this.#timer = undefined; + }, this.#timeout / 2); + } +} +function encodeAwarenessStates(states) { + const writer = new PostcardWriter(); + writer.writeUsize(states.length); + for (const state of states) { + writer.writeU64(state.peer); + writer.writeI32(state.counter); + writePostcardValue(writer, state.value); + } + return writer.toUint8Array(); +} +function decodeAwarenessStates(bytes) { + const reader = new PostcardReader(bytes); + const states = reader.readArray((input) => ({ + peer: input.readU64(), + counter: input.readI32(), + value: readPostcardValue(input), + })); + reader.assertEnd(); + return states; +} +function encodeEphemeralStates(states) { + const writer = new PostcardWriter(); + writer.writeUsize(states.length); + for (const state of states) { + writer.writeString(state.key); + writer.writeOption(state.value, writePostcardValue); + writer.writeI64(BigInt(state.timestamp)); + } + return writer.toUint8Array(); +} +function decodeEphemeralStates(bytes) { + const reader = new PostcardReader(bytes); + const states = reader.readArray((input) => ({ + key: input.readString(), + value: input.readOption(readPostcardValue), + timestamp: Number(input.readI64()), + })); + reader.assertEnd(); + return states; +} +function valueToEncoded(value, depth = 0) { + if (depth > 512) + throw new RangeError("LoroValue nesting depth exceeded"); + if (value === null || value === undefined) + return { type: "null" }; + if (typeof value === "boolean") + return { type: "bool", value }; + if (typeof value === "number") { + return Number.isInteger(value) && value >= -(2 ** 53) && value <= 2 ** 53 + ? { type: "i64", value: BigInt(value) } + : { type: "double", value }; + } + if (typeof value === "string") { + return isContainerId(value) + ? { type: "container", value: parseContainerId(value) } + : { type: "string", value }; + } + if (value instanceof Uint8Array) + return { type: "binary", value: value.slice() }; + if (Array.isArray(value)) { + return { type: "list", value: value.map((item) => valueToEncoded(item, depth + 1)) }; + } + if (value instanceof Map) { + return { + type: "map", + value: [...value].map(([key, item]) => { + if (typeof key !== "string") + throw new TypeError("Map keys must be strings"); + return [key, valueToEncoded(item, depth + 1)]; + }), + }; + } + if (typeof value === "object") { + return { + type: "map", + value: Object.entries(value).map(([key, item]) => [key, valueToEncoded(item, depth + 1)]), + }; + } + throw new TypeError(`unsupported LoroValue type: ${typeof value}`); +} +function encodedToValue(value) { + switch (value.type) { + case "null": + return null; + case "bool": + case "double": + case "string": + return value.value; + case "i64": + return Number(value.value); + case "binary": + return value.value.slice(); + case "list": + return value.value.map(encodedToValue); + case "map": + return Object.fromEntries(value.value.map(([key, item]) => [key, encodedToValue(item)])); + case "container": + return formatContainerId(value.value); + } +} +function assertTimeout(timeout) { + if (!Number.isFinite(timeout)) + throw new TypeError("timeout must be finite"); + return timeout; +} +function peerString(peer) { + return peer.toString(); +} +function errorMessage(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/loro-js/src/runtime/event-diff.js b/loro-js/src/runtime/event-diff.js new file mode 100644 index 000000000..e96ae5fc1 --- /dev/null +++ b/loro-js/src/runtime/event-diff.js @@ -0,0 +1,371 @@ +export class SequenceEventDiff { + #kind; + #originalLength; + #root; + #randomState = 2135587861; + constructor(kind, originalLength) { + this.#kind = kind; + this.#originalLength = originalLength; + this.#root = + originalLength === 0 + ? undefined + : this.#newNode({ + kind: "original", + start: 0, + length: originalLength, + attributes: undefined, + }); + } + get length() { + return nodeLength(this.#root); + } + insertText(position, value, attributes) { + if (this.#kind !== "text") + throw new Error("cannot insert text into a list diff"); + if (value.length === 0) + return; + const [left, right] = this.#splitAt(position); + const inserted = this.#newNode({ + kind: "text", + value, + attributes: attributes !== undefined && hasAttributes(attributes) ? attributes : undefined, + }); + this.#root = mergeNodes(mergeNodes(left, inserted), right); + } + insertList(position, value) { + if (this.#kind !== "list") + throw new Error("cannot insert a list into a text diff"); + if (value.length === 0) + return; + const [left, right] = this.#splitAt(position); + const inserted = this.#newNode({ kind: "list", value: [...value] }); + this.#root = mergeNodes(mergeNodes(left, inserted), right); + } + delete(position, length) { + if (length === 0) + return; + const [left, rest] = this.#splitAt(position); + this.#root = rest; + const [, right] = this.#splitAt(length); + this.#root = mergeNodes(left, right); + } + formatText(position, length, key, value) { + if (this.#kind !== "text") + throw new Error("cannot format a list diff"); + if (length === 0) + return; + const [left, rest] = this.#splitAt(position); + this.#root = rest; + const [selected, right] = this.#splitAt(length); + visitNodes(selected, (node) => { + const piece = node.piece; + if (piece.kind === "list") + throw new Error("text diff contains a list piece"); + const attributes = { ...piece.attributes, [key]: value }; + node.piece = + piece.kind === "original" + ? { + kind: "original", + start: piece.start, + length: piece.length, + attributes, + } + : { kind: "text", value: piece.value, attributes }; + }); + this.#root = mergeNodes(mergeNodes(left, selected), right); + } + toDiff() { + return this.#kind === "text" + ? { type: "text", diff: this.#textDelta() } + : { type: "list", diff: this.#listDelta() }; + } + #textDelta() { + const output = []; + let originalCursor = 0; + let inserted = []; + const flushInserted = () => { + let values = []; + let attributes; + const flushGroup = () => { + if (values.length === 0) + return; + appendTextDelta(output, { + insert: values.join(""), + ...(attributes === undefined ? {} : { attributes }), + }); + values = []; + }; + for (const piece of inserted) { + if (values.length > 0 && !attributesEqual(attributes, piece.attributes)) { + flushGroup(); + } + attributes = piece.attributes; + values.push(piece.value); + } + flushGroup(); + inserted = []; + }; + for (const piece of pieces(this.#root)) { + if (piece.kind === "text") { + inserted.push(piece); + continue; + } + if (piece.kind === "list") + throw new Error("text diff contains a list piece"); + if (piece.start < originalCursor) { + throw new Error("text diff original pieces are out of order"); + } + if (piece.start > originalCursor) { + appendTextDelta(output, { delete: piece.start - originalCursor }); + } + flushInserted(); + appendTextDelta(output, { + retain: piece.length, + ...(piece.attributes === undefined ? {} : { attributes: piece.attributes }), + }); + originalCursor = piece.start + piece.length; + } + if (originalCursor < this.#originalLength) { + appendTextDelta(output, { delete: this.#originalLength - originalCursor }); + } + flushInserted(); + trimPlainTrailingRetains(output); + return output; + } + #listDelta() { + const output = []; + let originalCursor = 0; + let inserted = []; + const flushInserted = () => { + if (inserted.length > 0) + appendListDelta(output, { insert: inserted }); + inserted = []; + }; + for (const piece of pieces(this.#root)) { + if (piece.kind === "list") { + inserted.push(...piece.value); + continue; + } + if (piece.kind === "text") + throw new Error("list diff contains a text piece"); + if (piece.start < originalCursor) { + throw new Error("list diff original pieces are out of order"); + } + if (piece.start > originalCursor) { + appendListDelta(output, { delete: piece.start - originalCursor }); + } + flushInserted(); + appendListDelta(output, { retain: piece.length }); + originalCursor = piece.start + piece.length; + } + if (originalCursor < this.#originalLength) { + appendListDelta(output, { delete: this.#originalLength - originalCursor }); + } + flushInserted(); + trimPlainTrailingRetains(output); + return output; + } + #splitAt(position) { + if (!Number.isSafeInteger(position) || position < 0 || position > this.length) { + throw new RangeError(`event diff position ${position} is out of range`); + } + const result = splitNode(this.#root, position, (piece) => this.#newNode(piece)); + this.#root = undefined; + return result; + } + #newNode(piece) { + this.#randomState ^= this.#randomState << 13; + this.#randomState ^= this.#randomState >>> 17; + this.#randomState ^= this.#randomState << 5; + return { + piece, + priority: this.#randomState >>> 0, + left: undefined, + right: undefined, + length: pieceLength(piece), + }; + } +} +function pieceLength(piece) { + if (piece.kind === "original") + return piece.length; + return piece.value.length; +} +function splitPiece(piece, position) { + if (piece.kind === "original") { + return [ + { + kind: "original", + start: piece.start, + length: position, + attributes: piece.attributes, + }, + { + kind: "original", + start: piece.start + position, + length: piece.length - position, + attributes: piece.attributes, + }, + ]; + } + if (piece.kind === "text") { + return [ + { kind: "text", value: piece.value.slice(0, position), attributes: piece.attributes }, + { kind: "text", value: piece.value.slice(position), attributes: piece.attributes }, + ]; + } + return [ + { kind: "list", value: piece.value.slice(0, position) }, + { kind: "list", value: piece.value.slice(position) }, + ]; +} +function nodeLength(node) { + return node?.length ?? 0; +} +function recomputeNode(node) { + node.length = nodeLength(node.left) + pieceLength(node.piece) + nodeLength(node.right); +} +function mergeNodes(left, right) { + if (left === undefined) + return right; + if (right === undefined) + return left; + if (left.priority <= right.priority) { + left.right = mergeNodes(left.right, right); + recomputeNode(left); + return left; + } + right.left = mergeNodes(left, right.left); + recomputeNode(right); + return right; +} +function splitNode(root, position, newNode) { + if (root === undefined) + return [undefined, undefined]; + const leftLength = nodeLength(root.left); + const ownLength = pieceLength(root.piece); + if (position < leftLength) { + const [left, right] = splitNode(root.left, position, newNode); + root.left = right; + recomputeNode(root); + return [left, root]; + } + if (position > leftLength + ownLength) { + const [left, right] = splitNode(root.right, position - leftLength - ownLength, newNode); + root.right = left; + recomputeNode(root); + return [root, right]; + } + if (position === leftLength) { + const left = root.left; + root.left = undefined; + recomputeNode(root); + return [left, root]; + } + if (position === leftLength + ownLength) { + const right = root.right; + root.right = undefined; + recomputeNode(root); + return [root, right]; + } + const [leftPiece, rightPiece] = splitPiece(root.piece, position - leftLength); + return [ + mergeNodes(root.left, newNode(leftPiece)), + mergeNodes(newNode(rightPiece), root.right), + ]; +} +function visitNodes(root, visit) { + const stack = []; + let node = root; + while (node !== undefined || stack.length > 0) { + while (node !== undefined) { + stack.push(node); + node = node.left; + } + node = stack.pop(); + visit(node); + node = node.right; + } +} +function pieces(root) { + const output = []; + visitNodes(root, (node) => output.push(node.piece)); + return output; +} +function appendTextDelta(output, operation) { + const previous = output.at(-1); + if (previous !== undefined && sameDeltaKind(previous, operation)) { + if ("delete" in previous && "delete" in operation) { + output[output.length - 1] = { delete: previous.delete + operation.delete }; + return; + } + if ("retain" in previous && + "retain" in operation && + attributesEqual(previous.attributes, operation.attributes)) { + output[output.length - 1] = { + retain: previous.retain + operation.retain, + ...(previous.attributes === undefined ? {} : { attributes: previous.attributes }), + }; + return; + } + if ("insert" in previous && + "insert" in operation && + attributesEqual(previous.attributes, operation.attributes)) { + output[output.length - 1] = { + insert: previous.insert + operation.insert, + ...(previous.attributes === undefined ? {} : { attributes: previous.attributes }), + }; + return; + } + } + output.push(operation); +} +function appendListDelta(output, operation) { + const previous = output.at(-1); + if (previous !== undefined && "delete" in previous && "delete" in operation) { + output[output.length - 1] = { delete: previous.delete + operation.delete }; + } + else if (previous !== undefined && "retain" in previous && "retain" in operation) { + output[output.length - 1] = { retain: previous.retain + operation.retain }; + } + else if (previous !== undefined && "insert" in previous && "insert" in operation) { + for (const item of operation.insert) + previous.insert.push(item); + } + else { + output.push(operation); + } +} +function sameDeltaKind(left, right) { + return (("insert" in left && "insert" in right) || + ("delete" in left && "delete" in right) || + ("retain" in left && "retain" in right)); +} +function attributesEqual(left, right) { + if (left === right) + return true; + if (left === undefined || right === undefined) + return false; + let count = 0; + for (const key in left) { + count += 1; + if (!Object.is(left[key], right[key])) + return false; + } + for (const _key in right) + count -= 1; + return count === 0; +} +function hasAttributes(attributes) { + for (const _key in attributes) + return true; + return false; +} +function trimPlainTrailingRetains(output) { + while (true) { + const last = output.at(-1); + if (last === undefined || !("retain" in last) || last.attributes !== undefined) + return; + output.pop(); + } +} diff --git a/loro-js/src/runtime/fractional-index.js b/loro-js/src/runtime/fractional-index.js new file mode 100644 index 000000000..facf17a84 --- /dev/null +++ b/loro-js/src/runtime/fractional-index.js @@ -0,0 +1,92 @@ +import { compareBytes } from "../codec/bytes"; +const TERMINATOR = 128; +export function fractionalIndexBetween(lower, upper) { + if (lower === undefined && upper === undefined) { + return Uint8Array.of(TERMINATOR); + } + if (lower === undefined) { + return terminated(newBefore(upper)); + } + if (upper === undefined) { + return terminated(newAfter(lower)); + } + if (compareBytes(lower, upper) >= 0) { + throw new RangeError("fractional index bounds are not ordered"); + } + const between = newBetween(lower, upper); + if (between === undefined) { + throw new RangeError("cannot create a fractional index between the bounds"); + } + return terminated(between); +} +function terminated(bytes) { + const output = new Uint8Array(bytes.length + 1); + output.set(bytes); + output[bytes.length] = TERMINATOR; + return output; +} +function newBefore(bytes) { + for (let index = 0; index < bytes.length; index += 1) { + if (bytes[index] > TERMINATOR) + return bytes.slice(0, index); + if (bytes[index] > 0) { + const output = bytes.slice(0, index + 1); + output[index] = output[index] - 1; + return output; + } + } + throw new RangeError("invalid fractional index"); +} +function newAfter(bytes) { + for (let index = 0; index < bytes.length; index += 1) { + if (bytes[index] < TERMINATOR) + return bytes.slice(0, index); + if (bytes[index] < 255) { + const output = bytes.slice(0, index + 1); + output[index] = output[index] + 1; + return output; + } + } + throw new RangeError("invalid fractional index"); +} +function newBetween(left, right) { + const shorterLength = Math.min(left.length, right.length) - 1; + for (let index = 0; index < shorterLength; index += 1) { + const difference = right[index] - left[index]; + if (difference > 1) { + const output = left.slice(0, index + 1); + output[index] = output[index] + Math.floor(difference / 2); + return output; + } + if (difference === 1) { + const tail = newAfter(left.subarray(index + 1)); + const output = new Uint8Array(index + 1 + tail.length); + output.set(left.subarray(0, index + 1)); + output.set(tail, index + 1); + return output; + } + if (difference < 0) + return undefined; + } + if (left.length < right.length) { + const split = shorterLength + 1; + if (right[split - 1] < TERMINATOR) + return undefined; + const tail = newBefore(right.subarray(split)); + const output = new Uint8Array(split + tail.length); + output.set(right.subarray(0, split)); + output.set(tail, split); + return output; + } + if (left.length > right.length) { + const split = shorterLength + 1; + if (left[split - 1] >= TERMINATOR) + return undefined; + const tail = newAfter(left.subarray(split)); + const output = new Uint8Array(split + tail.length); + output.set(left.subarray(0, split)); + output.set(tail, split); + return output; + } + return undefined; +} diff --git a/loro-js/src/runtime/ids.js b/loro-js/src/runtime/ids.js new file mode 100644 index 000000000..a82419d1e --- /dev/null +++ b/loro-js/src/runtime/ids.js @@ -0,0 +1,151 @@ +import { ContainerType as CodecContainerType, } from "../codec/types"; +const U64_MAX = 0xffffffffffffffffn; +export function parsePeerId(peer) { + let value; + try { + value = typeof peer === "bigint" ? peer : BigInt(peer); + } + catch { + throw new TypeError("peer id must be an unsigned 64-bit integer"); + } + if (value < 0n || value > U64_MAX) { + throw new RangeError(`peer id is out of range: ${String(peer)}`); + } + return value; +} +export function peerIdToString(peer) { + return peer.toString(); +} +export function parseOpId(id) { + assertCounter(id.counter); + return { peer: parsePeerId(id.peer), counter: id.counter }; +} +export function formatOpId(id) { + return { peer: peerIdToString(id.peer), counter: id.counter }; +} +export function formatTreeId(id) { + return `${id.counter}@${id.peer}`; +} +export function idStrToId(id) { + return formatOpId(parseTreeId(id)); +} +export function parseTreeId(id) { + const separator = id.indexOf("@"); + if (separator <= 0 || id.indexOf("@", separator + 1) !== -1) { + throw new TypeError(`invalid tree id: ${id}`); + } + const counter = Number(id.slice(0, separator)); + assertCounter(counter); + return { peer: parsePeerId(id.slice(separator + 1)), counter }; +} +export function codecTypeToPublic(type) { + switch (type) { + case CodecContainerType.Text: + return "Text"; + case CodecContainerType.Map: + return "Map"; + case CodecContainerType.List: + return "List"; + case CodecContainerType.Tree: + return "Tree"; + case CodecContainerType.MovableList: + return "MovableList"; + case CodecContainerType.Counter: + return "Counter"; + default: + throw new TypeError(`unsupported container type ${type.value}`); + } +} +export function publicTypeToCodec(type) { + switch (type) { + case "Text": + return CodecContainerType.Text; + case "Map": + return CodecContainerType.Map; + case "List": + return CodecContainerType.List; + case "Tree": + return CodecContainerType.Tree; + case "MovableList": + return CodecContainerType.MovableList; + case "Counter": + return CodecContainerType.Counter; + } +} +export function formatContainerId(id) { + const type = codecTypeToPublic(id.containerType); + return (id.kind === "root" + ? `cid:root-${id.name}:${type}` + : `cid:${id.counter}@${id.peer}:${type}`); +} +export function parseContainerId(id) { + if (!id.startsWith("cid:")) { + throw new TypeError(`invalid container id: ${id}`); + } + const typeSeparator = id.lastIndexOf(":"); + if (typeSeparator <= 4) { + throw new TypeError(`invalid container id: ${id}`); + } + const type = id.slice(typeSeparator + 1); + const containerType = publicTypeToCodec(type); + const body = id.slice(4, typeSeparator); + if (body.startsWith("root-")) { + return { kind: "root", name: body.slice(5), containerType }; + } + const operationId = parseTreeId(body); + return { kind: "normal", ...operationId, containerType }; +} +export function newContainerID(id, type) { + return formatContainerId({ + kind: "normal", + ...parseOpId(id), + containerType: publicTypeToCodec(type), + }); +} +export function newRootContainerID(name, type) { + return formatContainerId({ + kind: "root", + name, + containerType: publicTypeToCodec(type), + }); +} +export function isContainerId(value) { + // Cheap rejection first: allocating the TypeError (and its stack frames) is + // the dominant cost when this guards arbitrary strings. + if (!value.startsWith("cid:")) + return false; + try { + parseContainerId(value); + return true; + } + catch { + return false; + } +} +export function idsEqual(left, right) { + return left.peer === right.peer && left.counter === right.counter; +} +export function containerIdsEqual(left, right) { + // Structural comparison: stringifying both ids would allocate two throwaway + // strings per call on the B4 edit hot path. + if (left.kind !== right.kind || + !codecContainerTypesEqual(left.containerType, right.containerType)) { + return false; + } + return left.kind === "root" && right.kind === "root" + ? left.name === right.name + : left.kind === "normal" && + right.kind === "normal" && + left.peer === right.peer && + left.counter === right.counter; +} +function codecContainerTypesEqual(left, right) { + return typeof left === "string" || typeof right === "string" + ? left === right + : left.value === right.value; +} +function assertCounter(counter) { + if (!Number.isSafeInteger(counter) || counter < 0 || counter > 2147483647) { + throw new RangeError(`counter is out of range: ${counter}`); + } +} diff --git a/loro-js/src/runtime/index.js b/loro-js/src/runtime/index.js new file mode 100644 index 000000000..a3f1908d2 --- /dev/null +++ b/loro-js/src/runtime/index.js @@ -0,0 +1,8 @@ +export * from "./containers"; +export * from "./document"; +export * from "./ephemeral"; +export { idStrToId, isContainerId, newContainerID, newRootContainerID } from "./ids"; +export * from "./json-updates"; +export * from "./types"; +export * from "./undo"; +export * from "./version-vector"; diff --git a/loro-js/src/runtime/json-updates.js b/loro-js/src/runtime/json-updates.js new file mode 100644 index 000000000..8812984e1 --- /dev/null +++ b/loro-js/src/runtime/json-updates.js @@ -0,0 +1,60 @@ +export function redactJsonUpdates(input, versionRange) { + const parsed = typeof input === "string" ? JSON.parse(input) : input; + const schema = cloneJsonUpdateValue(parsed); + for (const change of schema.changes) { + const peerToken = change.id.slice(change.id.indexOf("@") + 1); + const peer = schema.peers === null ? peerToken : (schema.peers[Number(peerToken)] ?? peerToken); + const range = versionRange[peer]; + if (range === undefined) + continue; + for (const operation of change.ops) + redactOperation(operation, range); + } + return schema; +} +function redactOperation(operation, range) { + const content = operation.content; + if (content.type === "insert" && typeof content.text === "string") { + const characters = Array.from(content.text); + content.text = characters + .map((character, index) => inRange(operation.counter + index, range) ? "�" : character) + .join(""); + return; + } + if (content.type === "insert" && Array.isArray(content.value)) { + content.value = content.value.map((value, index) => inRange(operation.counter + index, range) ? redactValue(value) : value); + return; + } + if (!inRange(operation.counter, range)) + return; + if (content.type === "insert" && typeof content.key === "string") { + content.value = redactValue(content.value); + } + else if (content.type === "set") { + content.value = redactValue(content.value); + } + else if (content.type === "mark") { + content.style_value = null; + } +} +function redactValue(value) { + return typeof value === "string" && value.startsWith("🦜:") + ? value + : null; +} +function inRange(counter, range) { + return counter >= range[0] && counter < range[1]; +} +function cloneJsonUpdateValue(value) { + if (value instanceof Uint8Array) + return value.slice(); + if (Array.isArray(value)) + return value.map(cloneJsonUpdateValue); + if (typeof value === "object" && value !== null) { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, cloneJsonUpdateValue(item)])); + } + return value; +} +export function jsonUpdatePeer(schema, compressedPeer) { + return (schema.peers?.[Number(compressedPeer)] ?? compressedPeer); +} diff --git a/loro-js/src/runtime/jsonpath.js b/loro-js/src/runtime/jsonpath.js new file mode 100644 index 000000000..5e5c49504 --- /dev/null +++ b/loro-js/src/runtime/jsonpath.js @@ -0,0 +1,833 @@ +import { isContainer, LoroCounter, LoroList, LoroMap, LoroText, LoroTree, } from "./containers"; +export function evaluateJsonPath(root, query) { + const selectors = parseJsonPath(query); + let current = [root]; + for (const selector of selectors) { + current = selector.recursive + ? applyRecursiveSelector(current, selector, root) + : applySelector(current, selector, root); + } + return current; +} +const UNKNOWN_EVENT_INDEX = Symbol("unknownEventIndex"); +export function compileJsonPathEventMatcher(query) { + const steps = parseJsonPath(query).map((selector) => ({ + recursive: selector.recursive, + selectors: selector.kind === "child" + ? selector.keys.map((value) => ({ kind: "key", value })) + : selector.kind === "indices" + ? selector.indices.map((value) => value < 0 + ? { kind: "wildcard", value: undefined } + : { kind: "index", value }) + : [{ kind: "wildcard", value: undefined }], + })); + // Positions stay within [0, steps.length], so membership checks on the + // ping-pong arrays replace the former per-element Set + spread + sort. + const positionsAfter = (path, extraElement) => { + let positions = [0]; + let next = []; + const elementCount = extraElement === undefined ? path.length : path.length + 1; + for (let index = 0; index < elementCount; index += 1) { + const element = index < path.length ? path[index] : extraElement; + next.length = 0; + for (const position of positions) { + if (position >= steps.length) { + if (!next.includes(position)) + next.push(position); + continue; + } + const step = steps[position]; + if (step.recursive && !next.includes(position)) + next.push(position); + if (step.selectors.some((selector) => eventSelectorMatches(selector, element)) && + !next.includes(position + 1)) { + next.push(position + 1); + } + } + if (next.length === 0) + return next; + const swap = positions; + positions = next; + next = swap; + } + return positions; + }; + const mayMatch = (path) => steps.length === 0 || + positionsAfter(path).some((position) => position >= steps.length); + return (event) => { + const basePath = event.path; + if (mayMatch(basePath)) + return true; + if (event.diff.type === "map") { + const basePositions = positionsAfter(basePath); + if (basePositions.length === 0) + return false; + const passedWildcard = basePositions.some((position) => position > 0 && + steps[position - 1]?.selectors.some((selector) => selector.kind === "wildcard")); + for (const key in event.diff.updated) { + if (passedWildcard || positionsAfter(basePath, key).length > 0) + return true; + } + return false; + } + if (event.diff.type === "list" || + event.diff.type === "tree" || + event.diff.type === "counter") { + return positionsAfter(basePath, UNKNOWN_EVENT_INDEX).length > 0; + } + return false; + }; +} +function eventSelectorMatches(selector, element) { + if (selector.kind === "wildcard") + return true; + if (selector.kind === "key") + return typeof element === "string" && element === selector.value; + return element === UNKNOWN_EVENT_INDEX || element === selector.value; +} +function parseJsonPath(query) { + if (typeof query !== "string" || query.length === 0 || query[0] !== "$") { + throw new TypeError("JSONPath must start with '$'"); + } + const selectors = []; + let index = 1; + while (index < query.length) { + let recursive = false; + if (query.startsWith("..", index)) { + recursive = true; + index += 2; + } + else if (query[index] === ".") { + index += 1; + } + else if (query[index] !== "[") { + throw new SyntaxError(`unexpected JSONPath token at ${index}`); + } + if (query[index] === "[") { + const bracket = readBracket(query, index); + selectors.push(parseBracketSelector(bracket.content, recursive)); + index = bracket.end; + continue; + } + if (query[index] === "*") { + selectors.push({ kind: "wildcard", recursive }); + index += 1; + continue; + } + const start = index; + while (index < query.length && /[A-Za-z0-9_$-]/u.test(query[index])) { + index += 1; + } + if (index === start) + throw new SyntaxError(`missing JSONPath child at ${index}`); + selectors.push({ kind: "child", keys: [query.slice(start, index)], recursive }); + } + return selectors; +} +function readBracket(input, start) { + let depth = 0; + let quote; + let escaped = false; + for (let index = start; index < input.length; index += 1) { + const character = input[index]; + if (quote !== undefined) { + if (escaped) + escaped = false; + else if (character === "\\") + escaped = true; + else if (character === quote) + quote = undefined; + continue; + } + if (character === "'" || character === '"') { + quote = character; + continue; + } + if (character === "[") + depth += 1; + else if (character === "]") { + depth -= 1; + if (depth === 0) { + return { content: input.slice(start + 1, index), end: index + 1 }; + } + } + } + throw new SyntaxError(`unterminated JSONPath bracket at ${start}`); +} +function parseBracketSelector(content, recursive) { + const trimmed = content.trim(); + if (trimmed === "*") + return { kind: "wildcard", recursive }; + if (trimmed.startsWith("?(") && trimmed.endsWith(")")) { + return { + kind: "filter", + expression: trimmed.slice(2, -1), + recursive, + }; + } + const parts = splitTopLevel(trimmed, ","); + if (parts.length > 1) { + if (parts.every((part) => isQuoted(part.trim()))) { + return { + kind: "child", + keys: parts.map((part) => parseQuotedString(part.trim())), + recursive, + }; + } + if (parts.every((part) => /^-?\d+$/u.test(part.trim()))) { + return { + kind: "indices", + indices: parts.map((part) => Number(part.trim())), + recursive, + }; + } + throw new SyntaxError("JSONPath union must contain only keys or only indexes"); + } + if (isQuoted(trimmed)) { + return { kind: "child", keys: [parseQuotedString(trimmed)], recursive }; + } + if (/^-?\d+$/u.test(trimmed)) { + return { kind: "indices", indices: [Number(trimmed)], recursive }; + } + const slices = splitTopLevel(trimmed, ":"); + if (slices.length >= 2 && slices.length <= 3) { + const parsePart = (part) => { + const value = part.trim(); + if (value === "") + return undefined; + if (!/^-?\d+$/u.test(value)) + throw new SyntaxError("invalid JSONPath slice"); + return Number(value); + }; + return { + kind: "slice", + start: parsePart(slices[0]), + end: parsePart(slices[1]), + step: slices[2] === undefined ? undefined : parsePart(slices[2]), + recursive, + }; + } + throw new SyntaxError(`unsupported JSONPath selector [${content}]`); +} +function applyRecursiveSelector(values, selector, root) { + const output = []; + // `applySelector` never reads `selector.recursive`, so the selector passes + // through unchanged instead of being cloned per visited node. The ancestor + // set is reused in place: add on the way down, delete on the way up, which + // keeps the same path-scoped cycle detection. + const ancestors = new Set(); + const holder = [undefined]; + const visit = (value) => { + holder[0] = value; + output.push(...applySelector(holder, selector, root)); + const object = objectIdentity(value); + if (object === undefined) { + for (const child of childValues(value)) + visit(child); + return; + } + if (ancestors.has(object)) + return; + ancestors.add(object); + for (const child of childValues(value)) + visit(child); + ancestors.delete(object); + }; + for (const value of values) + visit(value); + return output; +} +function applySelector(values, selector, root) { + const output = []; + switch (selector.kind) { + case "child": + for (const value of values) { + for (const key of selector.keys) { + const child = lookupChild(value, key); + if (child.found) + output.push(child.value); + } + } + return output; + case "wildcard": + return values.flatMap(childValues); + case "indices": + for (const value of values) { + if (value instanceof LoroList) { + for (const rawIndex of selector.indices) { + const index = rawIndex < 0 ? value.length + rawIndex : rawIndex; + if (index >= 0 && index < value.length) + output.push(value.get(index)); + } + continue; + } + if (value instanceof LoroTree) { + for (const rawIndex of selector.indices) { + const selected = value._rootJsonValueAt(rawIndex); + if (selected !== undefined) + output.push(selected); + } + continue; + } + const children = sequenceValues(value); + if (children === undefined) + continue; + for (const rawIndex of selector.indices) { + const index = rawIndex < 0 ? children.length + rawIndex : rawIndex; + if (index >= 0 && index < children.length) + output.push(children[index]); + } + } + return output; + case "slice": + for (const value of values) { + if (value instanceof LoroList) { + output.push(...sliceIndexedValues(value.length, (index) => value.get(index), (start, end) => value._valuesRange(start, end), selector.start, selector.end, selector.step)); + continue; + } + if (value instanceof LoroTree) { + output.push(...sliceIndexedValues(value._rootCount(), (index) => value._rootJsonValueAt(index), (start, end) => value._rootJsonValuesRange(start, end), selector.start, selector.end, selector.step)); + continue; + } + const children = sequenceValues(value); + if (children === undefined) + continue; + output.push(...sliceValues(children, selector.start, selector.end, selector.step)); + } + return output; + case "filter": { + const expression = new FilterParser(selector.expression, root); + for (const value of values) { + for (const child of childValues(value)) { + if (expression.test(child)) + output.push(child); + } + } + return output; + } + } +} +function lookupChild(value, key) { + if (value instanceof LoroMap) { + const record = value._entries.get(key); + if (record === undefined || record.deleted) { + return { found: false, value: undefined }; + } + return { found: true, value: value.get(key) }; + } + if (value instanceof Map) { + return value.has(key) + ? { found: true, value: value.get(key) } + : { found: false, value: undefined }; + } + if (value instanceof LoroList) { + if (!/^-?\d+$/u.test(key)) + return { found: false, value: undefined }; + const rawIndex = Number(key); + const index = rawIndex < 0 ? value.length + rawIndex : rawIndex; + return index >= 0 && index < value.length + ? { found: true, value: value.get(index) } + : { found: false, value: undefined }; + } + if (value instanceof LoroTree) { + if (!/^-?\d+$/u.test(key)) + return { found: false, value: undefined }; + const selected = value._rootJsonValueAt(Number(key)); + return selected === undefined + ? { found: false, value: undefined } + : { found: true, value: selected }; + } + if (Array.isArray(value)) { + if (!/^-?\d+$/u.test(key)) + return { found: false, value: undefined }; + const rawIndex = Number(key); + const index = rawIndex < 0 ? value.length + rawIndex : rawIndex; + return index >= 0 && index < value.length + ? { found: true, value: value[index] } + : { found: false, value: undefined }; + } + if (typeof value === "object" && value !== null && !isContainer(value)) { + return Object.prototype.hasOwnProperty.call(value, key) + ? { found: true, value: value[key] } + : { found: false, value: undefined }; + } + return { found: false, value: undefined }; +} +function childValues(value) { + if (value instanceof LoroMap) + return value.values(); + if (value instanceof Map) + return [...value.values()]; + if (value instanceof LoroList) + return value.toArray(); + if (value instanceof LoroTree) + return value.toJSON(); + if (Array.isArray(value)) + return [...value]; + if (typeof value === "object" && value !== null && !isContainer(value)) { + return Object.values(value); + } + return []; +} +function sequenceValues(value) { + if (value instanceof LoroList) + return value.toArray(); + if (value instanceof LoroTree) + return value.toJSON(); + return Array.isArray(value) ? [...value] : undefined; +} +function sliceValues(values, rawStart, rawEnd, rawStep) { + const step = rawStep ?? 1; + if (step === 0) + return []; + const length = values.length; + const normalize = (value) => { + const resolved = value < 0 ? length + value : value; + return Math.max(step > 0 ? 0 : -1, Math.min(step > 0 ? length : length - 1, resolved)); + }; + const start = rawStart === undefined ? (step > 0 ? 0 : length - 1) : normalize(rawStart); + const end = rawEnd === undefined ? (step > 0 ? length : -1) : normalize(rawEnd); + const output = []; + if (step > 0) { + for (let index = start; index < end; index += step) + output.push(values[index]); + } + else { + for (let index = start; index > end; index += step) + output.push(values[index]); + } + return output; +} +function sliceIndexedValues(length, at, range, rawStart, rawEnd, rawStep) { + const step = rawStep ?? 1; + if (step === 0) + return []; + const normalize = (value) => { + const resolved = value < 0 ? length + value : value; + return Math.max(step > 0 ? 0 : -1, Math.min(step > 0 ? length : length - 1, resolved)); + }; + const start = rawStart === undefined ? (step > 0 ? 0 : length - 1) : normalize(rawStart); + const end = rawEnd === undefined ? (step > 0 ? length : -1) : normalize(rawEnd); + if (step === 1) + return range(start, end); + const output = []; + if (step > 0) { + for (let index = start; index < end; index += step) + output.push(at(index)); + } + else { + for (let index = start; index > end; index += step) + output.push(at(index)); + } + return output; +} +const COMPARISON_OPERATORS = ["==", "!=", "<", "<=", ">", ">=", "contains", "in"]; +const DOUBLE_CHAR_OPERATORS = ["&&", "||", "==", "!=", "<=", ">="]; +const SINGLE_CHAR_OPERATORS = ["!", "<", ">"]; +const PUNCTUATION_CHARS = ["(", ")", "[", "]", ","]; +const FILTER_PATH_TERMINATORS = [")", ",", "!", "<", ">", "="]; +class FilterParser { + #tokens; + #root; + #index = 0; + #current; + constructor(expression, root) { + this.#tokens = tokenizeFilter(expression); + this.#root = root; + } + test(current) { + this.#index = 0; + this.#current = current; + const value = this.#parseOr(); + this.#expect("eof"); + return filterTruthy(value); + } + #parseOr() { + let left = this.#parseAnd(); + while (this.#match("operator", "||")) { + const right = this.#parseAnd(); + left = filterTruthy(left) || filterTruthy(right); + } + return left; + } + #parseAnd() { + let left = this.#parseComparison(); + while (this.#match("operator", "&&")) { + const right = this.#parseComparison(); + left = filterTruthy(left) && filterTruthy(right); + } + return left; + } + #parseComparison() { + const left = this.#parseUnary(); + const token = this.#peek(); + if (token.kind !== "operator" || + !COMPARISON_OPERATORS.includes(token.value)) { + return left; + } + this.#index += 1; + return compareFilterValues(left, this.#parseUnary(), token.value); + } + #parseUnary() { + if (this.#match("operator", "!")) + return !filterTruthy(this.#parseUnary()); + return this.#parsePrimary(); + } + #parsePrimary() { + const token = this.#peek(); + if (token.kind === "literal") { + this.#index += 1; + return token.value; + } + if (token.kind === "path") { + this.#index += 1; + const root = token.value.startsWith("@") ? this.#current : this.#root; + const query = token.value.startsWith("@") + ? `$${token.value.slice(1)}` + : token.value; + return { + kind: "path-result", + values: evaluateJsonPath(root, query), + }; + } + if (token.kind === "identifier") { + this.#index += 1; + const name = token.value; + this.#expect("punctuation", "("); + const args = []; + if (!this.#match("punctuation", ")")) { + do + args.push(this.#parseOr()); + while (this.#match("punctuation", ",")); + this.#expect("punctuation", ")"); + } + return callFilterFunction(name, args); + } + if (this.#match("punctuation", "(")) { + const value = this.#parseOr(); + this.#expect("punctuation", ")"); + return value; + } + if (this.#match("punctuation", "[")) { + const values = []; + if (!this.#match("punctuation", "]")) { + do + values.push(this.#parseOr()); + while (this.#match("punctuation", ",")); + this.#expect("punctuation", "]"); + } + return values; + } + throw new SyntaxError(`unexpected filter token ${token.kind}:${token.value}`); + } + #peek() { + return this.#tokens[this.#index]; + } + #match(kind, value) { + const token = this.#peek(); + if (token.kind !== kind || (value !== undefined && token.value !== value)) + return false; + this.#index += 1; + return true; + } + #expect(kind, value) { + if (!this.#match(kind, value)) { + const token = this.#peek(); + throw new SyntaxError(`expected ${value ?? kind}, got ${token.kind}:${JSON.stringify(token.value)}`); + } + } +} +function tokenizeFilter(expression) { + const tokens = []; + let index = 0; + while (index < expression.length) { + const character = expression[index]; + if (/\s/u.test(character)) { + index += 1; + continue; + } + const pair = expression.slice(index, index + 2); + if (DOUBLE_CHAR_OPERATORS.includes(pair)) { + tokens.push({ kind: "operator", value: pair }); + index += 2; + continue; + } + if (SINGLE_CHAR_OPERATORS.includes(character)) { + tokens.push({ kind: "operator", value: character }); + index += 1; + continue; + } + if (PUNCTUATION_CHARS.includes(character)) { + tokens.push({ kind: "punctuation", value: character }); + index += 1; + continue; + } + if (character === "'" || character === '"') { + const literal = readQuotedLiteral(expression, index); + tokens.push({ kind: "literal", value: literal.value }); + index = literal.end; + continue; + } + if (character === "@" || character === "$") { + const path = readFilterPath(expression, index); + tokens.push({ kind: "path", value: path.value }); + index = path.end; + continue; + } + const number = expression.slice(index).match(/^-?(?:\d+(?:\.\d*)?|\.\d+)/u)?.[0]; + if (number !== undefined) { + tokens.push({ kind: "literal", value: Number(number) }); + index += number.length; + continue; + } + const identifier = expression.slice(index).match(/^[A-Za-z_][A-Za-z0-9_]*/u)?.[0]; + if (identifier !== undefined) { + if (identifier === "true" || identifier === "false") { + tokens.push({ kind: "literal", value: identifier === "true" }); + } + else if (identifier === "null") { + tokens.push({ kind: "literal", value: null }); + } + else if (identifier === "contains" || identifier === "in") { + tokens.push({ kind: "operator", value: identifier }); + } + else { + tokens.push({ kind: "identifier", value: identifier }); + } + index += identifier.length; + continue; + } + throw new SyntaxError(`invalid filter token at ${index}`); + } + tokens.push({ kind: "eof", value: "" }); + return tokens; +} +function readFilterPath(expression, start) { + let index = start + 1; + let bracketDepth = 0; + let quote; + let escaped = false; + while (index < expression.length) { + const character = expression[index]; + if (quote !== undefined) { + if (escaped) + escaped = false; + else if (character === "\\") + escaped = true; + else if (character === quote) + quote = undefined; + index += 1; + continue; + } + if (character === "'" || character === '"') { + quote = character; + index += 1; + continue; + } + if (character === "[") { + bracketDepth += 1; + index += 1; + continue; + } + if (character === "]" && bracketDepth > 0) { + bracketDepth -= 1; + index += 1; + continue; + } + if (bracketDepth === 0 && + (/\s/u.test(character) || FILTER_PATH_TERMINATORS.includes(character))) { + break; + } + index += 1; + } + return { value: expression.slice(start, index), end: index }; +} +function readQuotedLiteral(input, start) { + const quote = input[start]; + let escaped = false; + for (let index = start + 1; index < input.length; index += 1) { + const character = input[index]; + if (escaped) { + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character === quote) { + return { + value: parseQuotedString(input.slice(start, index + 1)), + end: index + 1, + }; + } + } + throw new SyntaxError(`unterminated string literal at ${start}`); +} +function compareFilterValues(left, right, operator) { + const leftValues = atomValues(left); + const rightValues = atomValues(right); + if (operator === "in") { + const candidates = rightValues.flatMap((value) => collectionValues(value)); + return leftValues.some((value) => candidates.some((candidate) => primitiveCompare(value, candidate, "=="))); + } + return leftValues.some((leftValue) => rightValues.some((rightValue) => primitiveCompare(leftValue, rightValue, operator))); +} +function primitiveCompare(left, right, operator) { + const a = comparableValue(left); + const b = comparableValue(right); + switch (operator) { + case "==": + return Object.is(a, b); + case "!=": + return !Object.is(a, b); + case "contains": + return typeof a === "string" + ? typeof b === "string" && a.includes(b) + : Array.isArray(a) && a.some((value) => Object.is(value, b)); + case "<": + if (typeof a === "number" && typeof b === "number") + return a < b; + if (typeof a === "string" && typeof b === "string") + return a < b; + return false; + case "<=": + if (typeof a === "number" && typeof b === "number") + return a <= b; + if (typeof a === "string" && typeof b === "string") + return a <= b; + return false; + case ">": + if (typeof a === "number" && typeof b === "number") + return a > b; + if (typeof a === "string" && typeof b === "string") + return a > b; + return false; + case ">=": + if (typeof a === "number" && typeof b === "number") + return a >= b; + if (typeof a === "string" && typeof b === "string") + return a >= b; + return false; + default: + throw new SyntaxError(`unsupported filter operator ${operator}`); + } +} +function callFilterFunction(name, args) { + if (name === "count") { + if (args.length !== 1) + throw new TypeError("count() expects one argument"); + return isPathResult(args[0]) + ? args[0].values.length + : collectionValues(args[0]).length; + } + if (name === "value") { + if (args.length !== 1) + throw new TypeError("value() expects one argument"); + return isPathResult(args[0]) ? args[0].values[0] : args[0]; + } + if (name === "length") { + if (args.length !== 1) + throw new TypeError("length() expects one argument"); + const value = isPathResult(args[0]) ? args[0].values[0] : args[0]; + if (typeof value === "string" || Array.isArray(value)) + return value.length; + if (value instanceof LoroMap) + return value.size; + if (value instanceof Map) + return value.size; + if (value instanceof LoroList) + return value.length; + if (typeof value === "object" && value !== null) + return Object.keys(value).length; + return 0; + } + throw new TypeError(`unsupported JSONPath function ${name}()`); +} +function atomValues(value) { + return isPathResult(value) ? [...value.values] : [value]; +} +function collectionValues(value) { + if (Array.isArray(value)) + return value; + if (value instanceof LoroList) + return value.toArray(); + if (value instanceof LoroMap) + return value.values(); + if (value instanceof Map) + return [...value.values()]; + return [value]; +} +function filterTruthy(value) { + if (isPathResult(value)) { + return (value.values.length > 0 && + value.values.some((item) => Boolean(comparableValue(item)))); + } + return Boolean(comparableValue(value)); +} +function comparableValue(value) { + if (value instanceof LoroText) + return value.toString(); + if (value instanceof LoroCounter) + return value.value; + return value; +} +function isPathResult(value) { + return (typeof value === "object" && + value !== null && + value.kind === "path-result"); +} +function objectIdentity(value) { + return typeof value === "object" && value !== null ? value : undefined; +} +function splitTopLevel(input, separator) { + const parts = []; + let start = 0; + let squareDepth = 0; + let roundDepth = 0; + let quote; + let escaped = false; + for (let index = 0; index < input.length; index += 1) { + const character = input[index]; + if (quote !== undefined) { + if (escaped) + escaped = false; + else if (character === "\\") + escaped = true; + else if (character === quote) + quote = undefined; + continue; + } + if (character === "'" || character === '"') + quote = character; + else if (character === "[") + squareDepth += 1; + else if (character === "]") + squareDepth -= 1; + else if (character === "(") + roundDepth += 1; + else if (character === ")") + roundDepth -= 1; + else if (character === separator && squareDepth === 0 && roundDepth === 0) { + parts.push(input.slice(start, index)); + start = index + 1; + } + } + parts.push(input.slice(start)); + return parts; +} +function isQuoted(value) { + return (value.length >= 2 && + ((value[0] === "'" && value.at(-1) === "'") || + (value[0] === '"' && value.at(-1) === '"'))); +} +function parseQuotedString(value) { + if (!isQuoted(value)) + throw new SyntaxError("invalid quoted JSONPath key"); + const quote = value[0]; + const body = value.slice(1, -1); + if (quote === '"') + return JSON.parse(value); + const doubleQuoted = `"${body.replace(/"/gu, '\\"').replace(/\\'/gu, "'")}"`; + return JSON.parse(doubleQuoted); +} diff --git a/loro-js/src/runtime/mergeable.js b/loro-js/src/runtime/mergeable.js new file mode 100644 index 000000000..c451f2344 --- /dev/null +++ b/loro-js/src/runtime/mergeable.js @@ -0,0 +1,124 @@ +import { containerTypeFromRawByte, containerTypeToRawByte, encodeContainerId, } from "../codec/container-id"; +import { ContainerType, } from "../codec/types"; +const textEncoder = new TextEncoder(); +const MERGEABLE_PREFIX = "🤝:"; +const MARKER_MAGIC = Uint8Array.of(0, 0x4c, 0x4d, 1); +const MARKER_DOMAIN = textEncoder.encode("loro.mergeable.marker.v1"); +export function newMergeableContainerId(parent, key, containerType) { + if (parent.containerType !== ContainerType.Map) { + throw new TypeError("mergeable child parent must be a map"); + } + const parentPayload = parent.kind === "root" && parent.name.startsWith(MERGEABLE_PREFIX) + ? parent.name.slice(MERGEABLE_PREFIX.length) + : parent.kind === "root" + ? `$${escapeSegment(parent.name)}` + : `@${parent.peer.toString(36)}:${signedBase36(parent.counter)}`; + return { + kind: "root", + name: `${MERGEABLE_PREFIX}${parentPayload}>${escapeSegment(key)}`, + containerType, + }; +} +export function mergeableMarker(parent, key, containerType) { + const rawType = containerTypeToRawByte(containerType); + const parentBytes = encodeContainerId(parent); + const keyBytes = textEncoder.encode(key); + const input = new Uint8Array(MARKER_DOMAIN.length + + varintLength(parentBytes.length) + + parentBytes.length + + varintLength(keyBytes.length) + + keyBytes.length + + 1); + let offset = MARKER_DOMAIN.length; + input.set(MARKER_DOMAIN, 0); + offset = writeVarint(input, offset, parentBytes.length); + input.set(parentBytes, offset); + offset += parentBytes.length; + offset = writeVarint(input, offset, keyBytes.length); + input.set(keyBytes, offset); + offset += keyBytes.length; + input[offset] = rawType; + const digest = crc32(input) & 16777215; + return Uint8Array.of(...MARKER_MAGIC, rawType, (digest >>> 16) & 0xff, (digest >>> 8) & 0xff, digest & 0xff); +} +export function parseMergeableMarker(parent, key, value) { + if (!(value instanceof Uint8Array) || value.length !== 8) + return undefined; + for (let index = 0; index < MARKER_MAGIC.length; index += 1) { + if (value[index] !== MARKER_MAGIC[index]) + return undefined; + } + const type = containerTypeFromRawByte(value[4]); + if (typeof type !== "string") + return undefined; + const expected = mergeableMarker(parent, key, type); + for (let index = 0; index < expected.length; index += 1) { + if (value[index] !== expected[index]) + return undefined; + } + return type; +} +export function isMergeableContainerId(id) { + return id.kind === "root" && id.name.startsWith(MERGEABLE_PREFIX); +} +function escapeSegment(value) { + let output = ""; + for (const character of value) { + if (character === "\\") + output += "\\\\"; + else if (character === ">") + output += "\\>"; + else if (character === "/") + output += "\\s"; + else if (character === "\0") + output += "\\0"; + else + output += character; + } + return output; +} +function signedBase36(value) { + return value < 0 ? `-${(-value).toString(36)}` : value.toString(36); +} +function varintLength(value) { + let length = 1; + while (value >= 0x80) { + value = Math.floor(value / 0x80); + length += 1; + } + return length; +} +function writeVarint(output, offset, value) { + do { + let byte = value & 0x7f; + value = Math.floor(value / 0x80); + if (value > 0) + byte |= 0x80; + output[offset] = byte; + offset += 1; + } while (value > 0); + return offset; +} +let crc32Table; +function getCrc32Table() { + if (crc32Table === undefined) { + const table = new Int32Array(256); + for (let byte = 0; byte < 256; byte += 1) { + let crc = byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (3988292384 & -(crc & 1)); + } + table[byte] = crc; + } + crc32Table = table; + } + return crc32Table; +} +function crc32(bytes) { + const table = getCrc32Table(); + let crc = 4294967295; + for (let index = 0; index < bytes.length; index += 1) { + crc = (crc >>> 8) ^ table[(crc ^ bytes[index]) & 0xff]; + } + return ~crc >>> 0; +} diff --git a/loro-js/src/runtime/ordered-index.js b/loro-js/src/runtime/ordered-index.js new file mode 100644 index 000000000..ce1a2c758 --- /dev/null +++ b/loro-js/src/runtime/ordered-index.js @@ -0,0 +1,303 @@ +export class OrderedIndex { + #root; + #last; + #randomState = 2246822507; + #compare; + #nodes = new WeakMap(); + constructor(compare) { + this.#compare = compare; + } + get size() { + return nodeSize(this.#root); + } + add(value) { + if (this.#nodes.has(value)) + throw new Error("ordered index already contains value"); + if (this.#last !== undefined && this.#compare(this.#last.value, value) < 0) { + const node = this.#newNode(value); + this.#nodes.set(value, node); + this.#root = merge(this.#root, node); + if (this.#root !== undefined) + this.#root.parent = undefined; + this.#last = node; + return; + } + const index = this.lowerBound(value); + const previous = this.at(index); + if (previous !== undefined && this.#compare(previous, value) === 0) { + throw new Error("ordered index keys must be unique"); + } + const node = this.#newNode(value); + this.#nodes.set(value, node); + const [left, right] = split(this.#root, index); + this.#root = merge(merge(left, node), right); + if (this.#root !== undefined) + this.#root.parent = undefined; + if (this.#last === undefined) + this.#last = node; + } + delete(value) { + const node = this.#nodes.get(value); + if (node === undefined) + return false; + const index = this.#indexOfNode(node); + if (index === undefined) + return false; + const [left, selectedAndRight] = split(this.#root, index); + const [, right] = split(selectedAndRight, 1); + this.#root = merge(left, right); + if (this.#root !== undefined) + this.#root.parent = undefined; + if (this.#last === node) { + this.#last = this.#root === undefined ? undefined : rightmost(this.#root); + } + this.#nodes.delete(value); + return true; + } + at(index) { + if (!Number.isSafeInteger(index) || index < 0 || index >= this.size) + return undefined; + let node = this.#root; + let remaining = index; + while (node !== undefined) { + const leftSize = nodeSize(node.left); + if (remaining < leftSize) + node = node.left; + else if (remaining === leftSize) + return node.value; + else { + remaining -= leftSize + 1; + node = node.right; + } + } + return undefined; + } + indexOf(value) { + const node = this.#nodes.get(value); + if (node === undefined) + return undefined; + return this.#indexOfNode(node); + } + #indexOfNode(node) { + let index = nodeSize(node.left); + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.right) { + index += nodeSize(current.parent.left) + 1; + } + current = current.parent; + } + return current === this.#root ? index : undefined; + } + lowerBound(value) { + return this._lowerBoundBy((current) => this.#compare(current, value)); + } + _lowerBoundBy(compare) { + let node = this.#root; + let offset = 0; + let answer = this.size; + while (node !== undefined) { + if (compare(node.value) < 0) { + offset += nodeSize(node.left) + 1; + node = node.right; + } + else { + answer = offset + nodeSize(node.left); + node = node.left; + } + } + return answer; + } + values() { + const output = []; + const stack = []; + let node = this.#root; + while (node !== undefined || stack.length > 0) { + while (node !== undefined) { + stack.push(node); + node = node.left; + } + node = stack.pop(); + output.push(node.value); + node = node.right; + } + return output; + } + valuesRange(start, end) { + const boundedStart = Math.max(0, Math.min(start, this.size)); + const boundedEnd = Math.max(boundedStart, Math.min(end, this.size)); + const output = []; + collectRange(this.#root, boundedStart, boundedEnd, output); + return output; + } + forEachFrom(lower, visit) { + const stack = []; + let node = this.#root; + while (node !== undefined) { + if (this.#compare(node.value, lower) < 0) { + node = node.right; + } + else { + stack.push(node); + node = node.left; + } + } + while (stack.length > 0) { + node = stack.pop(); + if (visit(node.value) === false) + return; + node = node.right; + while (node !== undefined) { + stack.push(node); + node = node.left; + } + } + } + /** + * In-order iteration starting at rank `index` in O(log n + visited). Skips + * the per-element `at(index)` treap walks of an index loop. An index at or + * beyond `size` visits nothing. + */ + _forEachFromIndex(index, visit) { + const stack = []; + let node = this.#root; + let remaining = index; + while (node !== undefined) { + const leftSize = nodeSize(node.left); + if (remaining < leftSize) { + stack.push(node); + node = node.left; + } + else if (remaining === leftSize) { + stack.push(node); + break; + } + else { + remaining -= leftSize + 1; + node = node.right; + } + } + while (stack.length > 0) { + node = stack.pop(); + if (visit(node.value) === false) + return; + node = node.right; + while (node !== undefined) { + stack.push(node); + node = node.left; + } + } + } + clear() { + this.#root = undefined; + this.#last = undefined; + this.#nodes = new WeakMap(); + } + #newNode(value) { + this.#randomState ^= this.#randomState << 13; + this.#randomState ^= this.#randomState >>> 17; + this.#randomState ^= this.#randomState << 5; + return { + value, + priority: this.#randomState >>> 0, + left: undefined, + right: undefined, + parent: undefined, + size: 1, + }; + } +} +function nodeSize(node) { + return node?.size ?? 0; +} +function rightmost(node) { + while (node.right !== undefined) + node = node.right; + return node; +} +function collectRange(node, start, end, output) { + if (node === undefined || start >= end) + return; + const leftSize = nodeSize(node.left); + if (start < leftSize) { + collectRange(node.left, start, Math.min(end, leftSize), output); + } + if (start <= leftSize && leftSize < end) + output.push(node.value); + if (end > leftSize + 1) { + collectRange(node.right, Math.max(0, start - leftSize - 1), end - leftSize - 1, output); + } +} +function recompute(node) { + node.size = nodeSize(node.left) + 1 + nodeSize(node.right); + if (node.left !== undefined) + node.left.parent = node; + if (node.right !== undefined) + node.right.parent = node; +} +function merge(left, right) { + if (left === undefined) { + if (right !== undefined) + right.parent = undefined; + return right; + } + if (right === undefined) { + left.parent = undefined; + return left; + } + if (left.priority <= right.priority) { + left.right = merge(left.right, right); + recompute(left); + left.parent = undefined; + return left; + } + right.left = merge(left, right.left); + recompute(right); + right.parent = undefined; + return right; +} +function split(root, count) { + if (root === undefined) + return [undefined, undefined]; + // Descend once, recording the spine. A spine node joins the right result + // when the split continues into its left subtree, the left result + // otherwise; the two carries are then stitched bottom-up. This keeps the + // recursive shape without allocating a [left, right] tuple per level. + const spine = []; + const intoLeftSubtree = []; + let node = root; + let remaining = count; + while (node !== undefined) { + const leftSize = nodeSize(node.left); + const intoLeft = leftSize >= remaining; + spine.push(node); + intoLeftSubtree.push(intoLeft); + if (intoLeft) { + node = node.left; + } + else { + remaining -= leftSize + 1; + node = node.right; + } + } + let left; + let right; + for (let index = spine.length - 1; index >= 0; index -= 1) { + const current = spine[index]; + if (intoLeftSubtree[index]) { + current.left = right; + recompute(current); + right = current; + } + else { + current.right = left; + recompute(current); + left = current; + } + } + if (left !== undefined) + left.parent = undefined; + if (right !== undefined) + right.parent = undefined; + return [left, right]; +} diff --git a/loro-js/src/runtime/sequence-deletion-index.js b/loro-js/src/runtime/sequence-deletion-index.js new file mode 100644 index 000000000..3c860a7cc --- /dev/null +++ b/loro-js/src/runtime/sequence-deletion-index.js @@ -0,0 +1,227 @@ +import { OrderedIndex } from "./ordered-index"; +// Reused deletion id passed to predicates so visited mappings do not each +// allocate a short-lived id object. Predicates must not retain the id +// argument or query deletions reentrantly. +const scratchDeletionId = { peer: 0n, counter: 0 }; +// Reused OrderedIndex.forEachFrom probe in `add`; only `.start` is read. +const targetSegmentProbe = { start: 0, end: 0, mappings: [] }; +export class SequenceDeletionIndex { + #targetsByPeer = new Map(); + #operationsByPeer = new Map(); + #serial = 0; + add(targetStart, length, deleteStart, direction) { + if (length <= 0) + return; + const mapping = { + targetPeer: targetStart.peer, + targetStart: targetStart.counter, + deletePeer: deleteStart.peer, + deleteStart: deleteStart.counter, + length, + direction, + }; + const targets = this.#targets(targetStart.peer); + const targetEnd = targetStart.counter + length; + this.#ensureBoundary(targets, targetStart.counter); + this.#ensureBoundary(targets, targetEnd); + const existing = []; + // OrderedIndex.forEachFrom only reads `.start` through the comparator + // and never retains the probe, so a shared scratch object is safe. + targetSegmentProbe.start = targetStart.counter; + targets.forEachFrom(targetSegmentProbe, (segment) => { + if (segment.start >= targetEnd) + return false; + existing.push(segment); + }); + let cursor = targetStart.counter; + for (const segment of existing) { + if (cursor < segment.start) { + targets.add({ start: cursor, end: segment.start, mappings: [mapping] }); + } + addMapping(segment.mappings, mapping); + cursor = segment.end; + } + if (cursor < targetEnd) { + targets.add({ start: cursor, end: targetEnd, mappings: [mapping] }); + } + this.#operations(deleteStart.peer).add({ + targetPeer: mapping.targetPeer, + targetStart: mapping.targetStart, + deletePeer: mapping.deletePeer, + deleteStart: mapping.deleteStart, + length: mapping.length, + direction: mapping.direction, + serial: this.#serial++, + }); + } + deletionIdsAt(target) { + const segment = this.#targetSegmentAt(target); + if (segment === undefined) + return []; + return segment.mappings.map((mapping) => ({ + peer: mapping.deletePeer, + counter: deleteCounterAt(mapping, target.counter), + })); + } + someDeletion(target, predicate) { + const mappings = this.#targetSegmentAt(target)?.mappings; + if (mappings === undefined) + return false; + for (let index = 0; index < mappings.length; index += 1) { + const mapping = mappings[index]; + scratchDeletionId.peer = mapping.deletePeer; + scratchDeletionId.counter = deleteCounterAt(mapping, target.counter); + if (predicate(scratchDeletionId)) + return true; + } + return false; + } + everyDeletion(target, predicate) { + const mappings = this.#targetSegmentAt(target)?.mappings; + if (mappings === undefined) + return true; + for (let index = 0; index < mappings.length; index += 1) { + const mapping = mappings[index]; + scratchDeletionId.peer = mapping.deletePeer; + scratchDeletionId.counter = deleteCounterAt(mapping, target.counter); + if (!predicate(scratchDeletionId)) + return false; + } + return true; + } + targetRunsDeletedBy(peer, start, end) { + if (start >= end) + return []; + const operations = this.#operationsByPeer.get(peer); + if (operations === undefined) + return []; + const selected = []; + const first = Math.max(0, operations._lowerBoundBy((operation) => operation.deleteStart - start) - 1); + for (let index = first; index < operations.size; index += 1) { + const operation = operations.at(index); + if (operation.deleteStart >= end) + break; + const overlapStart = Math.max(start, operation.deleteStart); + const overlapEnd = Math.min(end, operation.deleteStart + operation.length); + if (overlapStart >= overlapEnd) + continue; + const targetCounter = operation.direction === 1 + ? operation.targetStart + overlapStart - operation.deleteStart + : operation.targetStart + + operation.length - + (overlapEnd - operation.deleteStart); + selected.push({ + start: { peer: operation.targetPeer, counter: targetCounter }, + length: overlapEnd - overlapStart, + }); + } + return normalizeRuns(selected); + } + hasDeletionAt(runs, version) { + for (const run of runs) { + const targets = this.#targetsByPeer.get(run.start.peer); + if (targets === undefined) + continue; + const start = run.start.counter; + const end = start + run.length; + let index = Math.max(0, targets._lowerBoundBy((segment) => segment.start - (start + 1)) - 1); + for (; index < targets.size; index += 1) { + const segment = targets.at(index); + if (segment.start >= end) + break; + const overlapStart = Math.max(start, segment.start); + const overlapEnd = Math.min(end, segment.end); + if (overlapStart >= overlapEnd) + continue; + for (const mapping of segment.mappings) { + const minimumCounter = mapping.direction === 1 + ? deleteCounterAt(mapping, overlapStart) + : deleteCounterAt(mapping, overlapEnd - 1); + if (minimumCounter < (version.get(mapping.deletePeer) ?? 0)) + return true; + } + } + } + return false; + } + reset() { + this.#targetsByPeer.clear(); + this.#operationsByPeer.clear(); + this.#serial = 0; + } + #targets(peer) { + let targets = this.#targetsByPeer.get(peer); + if (targets === undefined) { + targets = new OrderedIndex((left, right) => left.start - right.start); + this.#targetsByPeer.set(peer, targets); + } + return targets; + } + #operations(peer) { + let operations = this.#operationsByPeer.get(peer); + if (operations === undefined) { + operations = new OrderedIndex((left, right) => left.deleteStart - right.deleteStart || left.serial - right.serial); + this.#operationsByPeer.set(peer, operations); + } + return operations; + } + #targetSegmentAt(target) { + const targets = this.#targetsByPeer.get(target.peer); + if (targets === undefined) + return undefined; + const index = targets._lowerBoundBy((segment) => segment.start - (target.counter + 1)) - 1; + const segment = targets.at(index); + return segment !== undefined && target.counter < segment.end ? segment : undefined; + } + #ensureBoundary(targets, counter) { + const index = targets._lowerBoundBy((segment) => segment.start - counter); + if (targets.at(index)?.start === counter) + return; + const previous = targets.at(index - 1); + if (previous === undefined || counter >= previous.end) + return; + targets.delete(previous); + const right = { + start: counter, + end: previous.end, + mappings: [...previous.mappings], + }; + previous.end = counter; + targets.add(previous); + targets.add(right); + } +} +function addMapping(mappings, mapping) { + if (mappings.some((current) => current.deletePeer === mapping.deletePeer && + current.deleteStart === mapping.deleteStart && + current.targetPeer === mapping.targetPeer && + current.targetStart === mapping.targetStart)) { + return; + } + mappings.push(mapping); +} +function deleteCounterAt(mapping, targetCounter) { + const offset = targetCounter - mapping.targetStart; + return (mapping.deleteStart + (mapping.direction === 1 ? offset : mapping.length - 1 - offset)); +} +function normalizeRuns(runs) { + const sorted = [...runs].sort((left, right) => left.start.peer < right.start.peer + ? -1 + : left.start.peer > right.start.peer + ? 1 + : left.start.counter - right.start.counter); + const merged = []; + for (const run of sorted) { + const previous = merged.at(-1); + if (previous !== undefined && + previous.start.peer === run.start.peer && + run.start.counter <= previous.start.counter + previous.length) { + previous.length = + Math.max(previous.start.counter + previous.length, run.start.counter + run.length) - previous.start.counter; + } + else { + merged.push({ start: { ...run.start }, length: run.length }); + } + } + return merged; +} diff --git a/loro-js/src/runtime/sequence-index.js b/loro-js/src/runtime/sequence-index.js new file mode 100644 index 000000000..37c924a55 --- /dev/null +++ b/loro-js/src/runtime/sequence-index.js @@ -0,0 +1,3173 @@ +import { OrderedIndex } from "./ordered-index"; +import { SequenceDeletionIndex } from "./sequence-deletion-index"; +/** + * A compact physical sequence span. Implementations can keep scalar fields in + * columns and only create a T view when an API actually asks for one. + */ +export class SequenceSpan { + peerAt(offset) { + return this.idAt(offset).peer; + } + counterAt(offset) { + return this.idAt(offset).counter; + } + idsAreUnique() { + return false; + } + append(_other) { + return false; + } + deletionIdsAt(_offset) { + return EMPTY_SEQUENCE_IDS; + } + retain(_offset, _element) { } + forEachRetained(_visit) { } +} +const SEQUENCE_NODE = Symbol("sequenceNode"); +const SEQUENCE_OFFSET = Symbol("sequenceOffset"); +const MAX_CACHED_CAUSAL_VIEWS = 8; +const MAX_SEQUENCE_SPAN = 32; +const SEQUENCE_LOCATION_STRIDE = 64; +// Shared list metrics: every receiver reads utf16/utf8 and never mutates. +const LIST_METRICS = { utf16: 1, utf8: 1 }; +const listMetrics = () => LIST_METRICS; +// Shared empty list for spans without deletion metadata; callers only read it. +const EMPTY_SEQUENCE_IDS = []; +export class SequenceIndex { + #root; + #tail; + #structureVersion = 0; + #randomState = 2654435769; + #metrics; + #locationsByPeer = new Map(); + #locationsByLamportPeer = new Map(); + #nodesByLocationId = []; + #deletionsByPeer = new Map(); + #scalarDeletionTargetsByPeer = new Map(); + #deletionRuns = new SequenceDeletionIndex(); + #maxCounterByPeer = new Map(); + #cachedCausalViews = []; + #hasLazyDeletions = false; + constructor(metrics = listMetrics) { + this.#metrics = metrics; + } + get allLength() { + return allCount(this.#root); + } + get visibleLength() { + return visibleCount(this.#root); + } + get visibleUtf16Length() { + return visibleMetric(this.#root, "utf16"); + } + get visibleUtf8Length() { + return visibleMetric(this.#root, "utf8"); + } + get spanCount() { + let count = 0; + visitInOrder(this.#root, () => { + count += 1; + }); + return count; + } + /** Changes whenever the physical order or membership changes. */ + get structureVersion() { + return this.#structureVersion; + } + all() { + if (this.#hasLazyDeletions) { + materializeAllDeletions(this.#root, this.#metrics); + this.#hasLazyDeletions = false; + } + const output = []; + visitInOrder(this.#root, (node) => appendNodeElements(node, output)); + return output; + } + visible() { + const output = []; + this.forEachVisible((element) => { + output.push(element); + }); + return output; + } + forEachVisible(visit) { + if (this.visibleLength === 0) + return; + const stack = []; + let node = this.#root; + while (node !== undefined || stack.length > 0) { + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + stack.push(node); + node = visibleCount(node.left) > 0 ? node.left : undefined; + } + node = stack.pop(); + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (nodeDeleted(node, offset)) + continue; + if (visit(nodeElement(node, offset)) === false) + return; + } + node = visibleCount(node.right) > 0 ? node.right : undefined; + } + } + /** + * Visits maximal visible ranges in each physical storage block without + * materializing scalar views from compact spans. + */ + forEachVisibleStorageRange(visit) { + if (this.visibleLength === 0) + return; + const stack = []; + let node = this.#root; + while (node !== undefined || stack.length > 0) { + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + stack.push(node); + node = visibleCount(node.left) > 0 ? node.left : undefined; + } + node = stack.pop(); + let rangeStart = -1; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (!nodeDeleted(node, offset)) { + if (rangeStart < 0) + rangeStart = offset; + continue; + } + if (rangeStart >= 0) { + if (visit(node.element, rangeStart, offset) === false) + return; + rangeStart = -1; + } + } + if (rangeStart >= 0 && + visit(node.element, rangeStart, nodeLength(node)) === false) { + return; + } + node = visibleCount(node.right) > 0 ? node.right : undefined; + } + } + visibleRange(start, end) { + const output = []; + this.forEachVisibleRange(start, end, (element) => { + output.push(element); + }); + return output; + } + forEachVisibleRange(start, end, visit) { + const boundedStart = Math.max(0, Math.min(start, this.visibleLength)); + const boundedEnd = Math.max(boundedStart, Math.min(end, this.visibleLength)); + visitVisibleRange(this.#root, boundedStart, boundedEnd, this.#metrics, visit); + } + someVisibleRange(start, end, predicate) { + let matched = false; + this.forEachVisibleRange(start, end, (element) => { + if (!predicate(element)) + return; + matched = true; + return false; + }); + return matched; + } + visibleIdRuns(start, end) { + const boundedStart = Math.max(0, Math.min(start, this.visibleLength)); + const boundedEnd = Math.max(boundedStart, Math.min(end, this.visibleLength)); + const runs = []; + const append = (id, length = 1) => { + if (length === 0) + return; + const previous = runs[runs.length - 1]; + if (previous !== undefined && + previous.start.peer === id.peer && + previous.start.counter + previous.length === id.counter) { + previous.length += length; + } + else { + runs.push({ start: { ...id }, length }); + } + }; + visitVisibleIdRuns(this.#root, boundedStart, boundedEnd, this.#metrics, append); + return runs; + } + visibleMetricRangesForIdRuns(runs, metric) { + const targets = counterRangesByPeer(runs); + const ranges = []; + const append = (start, end) => { + if (start >= end) + return; + const previous = ranges.at(-1); + if (previous?.end === start) + previous.end = end; + else + ranges.push({ start, end }); + }; + visitVisibleMetricRangesForIds(this.#root, 0, metric, targets, this.#metrics, append); + return ranges; + } + containsIdRuns(runs) { + if (runs.length === 0) + return true; + const targets = counterRangesByPeer(runs); + const expected = [...targets.values()].reduce((total, ranges) => total + ranges.reduce((subtotal, range) => subtotal + range.end - range.start, 0), 0); + return countPhysicalIdsInRanges(this.#root, targets) === expected; + } + atPhysical(index) { + const element = this.atPhysicalRaw(index); + if (element !== undefined && this.#hasLazyDeletions) { + materializeElementDeletion(element, this.#metrics); + } + return element; + } + atPhysicalRaw(index) { + if (!Number.isSafeInteger(index) || index < 0 || index >= this.allLength) { + return undefined; + } + let node = this.#root; + let remaining = index; + while (node !== undefined) { + const leftCount = allCount(node.left); + if (remaining < leftCount) { + node = node.left; + } + else if (remaining < leftCount + nodeLength(node)) { + return nodeElement(node, remaining - leftCount); + } + else { + remaining -= leftCount + nodeLength(node); + node = node.right; + } + } + return undefined; + } + forEachPhysicalRaw(visit) { + let index = 0; + let stopped = false; + visitInOrder(this.#root, (node) => { + if (stopped) + return; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (visit(nodeElement(node, offset), index) === false) { + stopped = true; + return; + } + index += 1; + } + }); + } + findNextIncludedPhysical(start, version) { + if (!Number.isSafeInteger(start) || start < 0 || start >= this.allLength) { + return undefined; + } + return findNextIncludedPhysical(this.#root, 0, start, version); + } + atVisible(index) { + if (!Number.isSafeInteger(index) || index < 0 || index >= this.visibleLength) { + return undefined; + } + let node = this.#root; + let remaining = index; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftCount = visibleCount(node.left); + if (remaining < leftCount) { + node = node.left; + continue; + } + remaining -= leftCount; + if (remaining < ownVisibleCount(node)) { + return nodeElement(node, physicalOffsetAtVisibleIndex(node, remaining)); + } + remaining -= ownVisibleCount(node); + node = node.right; + } + return undefined; + } + visibleInsertionContext(position) { + if (!Number.isSafeInteger(position) || + position < 0 || + position > this.visibleLength) { + throw new RangeError(`visible sequence position ${position} is out of range`); + } + if (position === 0) { + return { + current: true, + left: undefined, + startIndex: 0, + right: firstPhysicalElement(this.#root), + }; + } + let node = this.#root; + let remaining = position - 1; + let physicalBase = 0; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftPhysicalCount = allCount(node.left); + const leftVisibleCount = visibleCount(node.left); + if (remaining < leftVisibleCount) { + node = node.left; + continue; + } + remaining -= leftVisibleCount; + physicalBase += leftPhysicalCount; + if (remaining < ownVisibleCount(node)) { + const offset = physicalOffsetAtVisibleIndex(node, remaining); + const left = nodeElement(node, offset); + return { + current: true, + left, + startIndex: physicalBase + offset + 1, + right: nextPhysicalElement(node, offset), + }; + } + remaining -= ownVisibleCount(node); + physicalBase += nodeLength(node); + node = node.right; + } + throw new Error("visible sequence insertion position is missing"); + } + /** Writes insertion neighbor IDs without creating compact-span views. */ + visibleInsertionIdContext(position, output) { + if (!Number.isSafeInteger(position) || + position < 0 || + position > this.visibleLength) { + throw new RangeError(`visible sequence position ${position} is out of range`); + } + if (position === 0) { + output.leftPeer = undefined; + output.leftCounter = 0; + output.startIndex = 0; + writeFirstPhysicalId(this.#root, output); + return output; + } + let node = this.#root; + let remaining = position - 1; + let physicalBase = 0; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftPhysicalCount = allCount(node.left); + const leftVisibleCount = visibleCount(node.left); + if (remaining < leftVisibleCount) { + node = node.left; + continue; + } + remaining -= leftVisibleCount; + physicalBase += leftPhysicalCount; + if (remaining < ownVisibleCount(node)) { + const offset = physicalOffsetAtVisibleIndex(node, remaining); + output.leftPeer = nodePeer(node, offset); + output.leftCounter = nodeCounter(node, offset); + output.startIndex = physicalBase + offset + 1; + writeNextPhysicalId(node, offset, output); + return output; + } + remaining -= ownVisibleCount(node); + physicalBase += nodeLength(node); + node = node.right; + } + throw new Error("visible sequence insertion position is missing"); + } + findById(id) { + const element = this.findByIdRaw(id); + if (element !== undefined && this.#hasLazyDeletions) { + materializeElementDeletion(element, this.#metrics); + } + return element; + } + findByIdRaw(id) { + const location = this.#locationsByPeer.get(id.peer)?.get(id.counter); + return location === undefined + ? undefined + : typeof location === "number" + ? this.#elementAtLocation(location) + : location; + } + findByLamport(peer, lamport) { + let lamports = this.#locationsByLamportPeer.get(peer); + if (lamports === undefined) { + const locations = this.#locationsByPeer.get(peer); + if (locations === undefined) + return undefined; + const indexedLamports = new Map(); + locations.forEach(0, Number.MAX_SAFE_INTEGER, (location) => { + const lamport = typeof location === "number" + ? nodeLamport(this.#nodeAtLocation(location), location % SEQUENCE_LOCATION_STRIDE) + : location.lamport; + if (lamport !== undefined) { + indexedLamports.set(lamport, location); + } + }); + lamports = indexedLamports; + this.#locationsByLamportPeer.set(peer, lamports); + } + const location = lamports.get(lamport); + const element = location === undefined + ? undefined + : typeof location === "number" + ? this.#elementAtLocation(location) + : location; + if (element !== undefined && this.#hasLazyDeletions) { + materializeElementDeletion(element, this.#metrics); + } + return element; + } + physicalIndexOf(element) { + const node = elementNode(element); + const offset = elementOffset(element); + if (node === undefined || offset === undefined) + return undefined; + let index = allCount(node.left) + offset; + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.right) { + index += allCount(current.parent.left) + nodeLength(current.parent); + } + current = current.parent; + } + return current === this.#root ? index : undefined; + } + visibleIndexOf(element) { + if (this.#hasLazyDeletions) + materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + const offset = elementOffset(element); + if (node === undefined || offset === undefined) + return undefined; + let index = visibleCount(node.left) + visibleElementsBefore(node, offset); + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.right) { + index += visibleCount(current.parent.left) + ownVisibleCount(current.parent); + } + current = current.parent; + } + return current === this.#root ? index : undefined; + } + visibleMetricOffsetOf(element, metric) { + if (this.#hasLazyDeletions) + materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + const elementPhysicalOffset = elementOffset(element); + if (node === undefined || elementPhysicalOffset === undefined) + return undefined; + let offset = visibleMetric(node.left, metric) + + visibleMetricBefore(node, elementPhysicalOffset, metric, this.#metrics); + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.right) { + offset += + visibleMetric(current.parent.left, metric) + + ownVisibleMetric(current.parent, metric, this.#metrics); + } + current = current.parent; + } + return current === this.#root ? offset : undefined; + } + metricOffsetAtVisibleIndex(index, metric) { + if (!Number.isSafeInteger(index) || index < 0 || index > this.visibleLength) { + return undefined; + } + if (index === this.visibleLength) + return visibleMetric(this.#root, metric); + let node = this.#root; + let remaining = index; + let offset = 0; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftCount = visibleCount(node.left); + if (remaining < leftCount) { + node = node.left; + continue; + } + remaining -= leftCount; + offset += visibleMetric(node.left, metric); + if (remaining < ownVisibleCount(node)) { + const ownOffset = physicalOffsetAtVisibleIndex(node, remaining); + return offset + visibleMetricBefore(node, ownOffset, metric, this.#metrics); + } + remaining -= ownVisibleCount(node); + offset += ownVisibleMetric(node, metric, this.#metrics); + node = node.right; + } + return undefined; + } + visibleIndexAtMetricOffset(offset, metric) { + const total = visibleMetric(this.#root, metric); + if (!Number.isSafeInteger(offset) || offset < 0 || offset > total) + return undefined; + if (offset === total) + return this.visibleLength; + let node = this.#root; + let remaining = offset; + let index = 0; + while (node !== undefined) { + pushNodeDeletion(node, this.#metrics); + const leftMetric = visibleMetric(node.left, metric); + if (remaining < leftMetric) { + node = node.left; + continue; + } + remaining -= leftMetric; + index += visibleCount(node.left); + const ownMetric = ownVisibleMetric(node, metric, this.#metrics); + if (remaining <= ownMetric) { + const ownOffset = physicalOffsetAtMetricOffset(node, remaining, metric); + return ownOffset === undefined + ? undefined + : index + visibleElementsBefore(node, ownOffset); + } + remaining -= ownMetric; + index += ownVisibleCount(node); + node = node.right; + } + return undefined; + } + insertAtVisible(position, elements) { + if (position < 0 || position > this.visibleLength) { + throw new RangeError(`visible sequence position ${position} is out of range`); + } + const physical = position === this.visibleLength + ? this.allLength + : this.physicalIndexOf(this.atVisible(position)); + this.insertAtPhysical(physical, elements); + } + insertSpanAtVisible(position, span) { + if (position < 0 || position > this.visibleLength) { + throw new RangeError(`visible sequence position ${position} is out of range`); + } + const physical = position === this.visibleLength + ? this.allLength + : this.physicalIndexOf(this.atVisible(position)); + this.insertSpanAtPhysical(physical, span); + } + insertAtPhysical(position, elements) { + if (elements.length === 0) + return; + if (position < 0 || position > this.allLength) { + throw new RangeError(`physical sequence position ${position} is out of range`); + } + if (elements.length === 1) { + const id = elements[0].id; + if (this.#locationsByPeer.get(id.peer)?.has(id.counter) === true) { + throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); + } + } + else if (elements.length <= MAX_SEQUENCE_SPAN) { + for (let index = 0; index < elements.length; index += 1) { + const id = elements[index].id; + if (this.#locationsByPeer.get(id.peer)?.has(id.counter) === true) { + throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); + } + for (let previous = 0; previous < index; previous += 1) { + const other = elements[previous].id; + if (other.peer === id.peer && other.counter === id.counter) { + throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); + } + } + } + } + else { + // Same-peer strictly increasing ids cannot duplicate in-batch, so the + // common contiguous batch skips the string-keyed Set. + let idsAreOrdered = true; + let previousPeer = elements[0].id.peer; + let previousCounter = elements[0].id.counter; + for (let index = 1; index < elements.length; index += 1) { + const id = elements[index].id; + if (id.peer !== previousPeer || id.counter <= previousCounter) { + idsAreOrdered = false; + break; + } + previousPeer = id.peer; + previousCounter = id.counter; + } + if (idsAreOrdered) { + for (const element of elements) { + const id = element.id; + if (this.#locationsByPeer.get(id.peer)?.has(id.counter) === true) { + throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); + } + } + } + else { + const ids = new Set(); + for (const element of elements) { + const key = `${element.id.peer}:${element.id.counter}`; + if (ids.has(key) || + this.#locationsByPeer.get(element.id.peer)?.has(element.id.counter) === true) { + throw new Error(`duplicate sequence id ${element.id.counter}@${element.id.peer.toString()}`); + } + ids.add(key); + } + } + } + this.#invalidateCausalView(); + if (this.#appendToPreviousNode(position, elements)) { + this.#structureVersion += 1; + return; + } + let insertedRoot; + if (elements.length <= MAX_SEQUENCE_SPAN) { + insertedRoot = this.#newNode(elements, true); + } + else { + for (let start = 0; start < elements.length; start += MAX_SEQUENCE_SPAN) { + insertedRoot = merge(insertedRoot, this.#newNode(elements.slice(start, start + MAX_SEQUENCE_SPAN), true), this.#metrics); + } + } + const appendAtTail = position === this.allLength; + const previousTailElement = appendAtTail || this.#tail === undefined + ? undefined + : nodeElement(this.#tail, nodeLength(this.#tail) - 1); + if (appendAtTail) { + this.#root = this.#appendAtTail(this.#root, insertedRoot); + } + else if (insertedRoot !== undefined && + insertedRoot.left === undefined && + insertedRoot.right === undefined) { + this.#root = this.#insertSingleNode(this.#root, position, insertedRoot); + } + else { + const [left, right] = this.#split(this.#root, position); + this.#root = merge(merge(left, insertedRoot, this.#metrics), right, this.#metrics); + } + if (this.#root !== undefined) + this.#root.parent = undefined; + if (previousTailElement !== undefined) { + this.#tail = elementNode(previousTailElement); + } + this.#structureVersion += 1; + } + insertSpanAtPhysical(position, span) { + if (span.length === 0) + return; + if (position < 0 || position > this.allLength) { + throw new RangeError(`physical sequence position ${position} is out of range`); + } + if (span.length === 1) { + const peer = span.peerAt(0); + const counter = span.counterAt(0); + if (this.#locationsByPeer.get(peer)?.has(counter) === true) { + throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); + } + } + else if (span.idsAreUnique()) { + for (let offset = 0; offset < span.length; offset += 1) { + const peer = span.peerAt(offset); + const counter = span.counterAt(offset); + if (this.#locationsByPeer.get(peer)?.has(counter) === true) { + throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); + } + } + } + else if (span.length <= MAX_SEQUENCE_SPAN) { + for (let offset = 0; offset < span.length; offset += 1) { + const peer = span.peerAt(offset); + const counter = span.counterAt(offset); + if (this.#locationsByPeer.get(peer)?.has(counter) === true) { + throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); + } + for (let previous = 0; previous < offset; previous += 1) { + if (span.peerAt(previous) === peer && span.counterAt(previous) === counter) { + throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); + } + } + } + } + else { + // Same-peer strictly increasing ids cannot duplicate in-batch, so the + // common contiguous span skips the string-keyed Set. + let idsAreOrdered = true; + let previousPeer = span.peerAt(0); + let previousCounter = span.counterAt(0); + for (let offset = 1; offset < span.length; offset += 1) { + const peer = span.peerAt(offset); + const counter = span.counterAt(offset); + if (peer !== previousPeer || counter <= previousCounter) { + idsAreOrdered = false; + break; + } + previousPeer = peer; + previousCounter = counter; + } + if (idsAreOrdered) { + for (let offset = 0; offset < span.length; offset += 1) { + const peer = span.peerAt(offset); + const counter = span.counterAt(offset); + if (this.#locationsByPeer.get(peer)?.has(counter) === true) { + throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); + } + } + } + else { + const ids = new Set(); + for (let offset = 0; offset < span.length; offset += 1) { + const peer = span.peerAt(offset); + const counter = span.counterAt(offset); + const key = `${peer}:${counter}`; + if (ids.has(key) || this.#locationsByPeer.get(peer)?.has(counter) === true) { + throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); + } + ids.add(key); + } + } + } + this.#invalidateCausalView(); + if (this.#appendSpanToPreviousNode(position, span)) { + this.#structureVersion += 1; + return; + } + let insertedRoot; + if (span.length <= MAX_SEQUENCE_SPAN) { + insertedRoot = this.#newNode(span, true); + } + else { + for (let start = 0; start < span.length; start += MAX_SEQUENCE_SPAN) { + insertedRoot = merge(insertedRoot, this.#newNode(span.slice(start, Math.min(span.length, start + MAX_SEQUENCE_SPAN)), true), this.#metrics); + } + } + const appendAtTail = position === this.allLength; + const previousTailElement = appendAtTail || this.#tail === undefined + ? undefined + : nodeElement(this.#tail, nodeLength(this.#tail) - 1); + if (appendAtTail) { + this.#root = this.#appendAtTail(this.#root, insertedRoot); + } + else if (insertedRoot !== undefined && + insertedRoot.left === undefined && + insertedRoot.right === undefined) { + this.#root = this.#insertSingleNode(this.#root, position, insertedRoot); + } + else { + const [left, right] = this.#split(this.#root, position); + this.#root = merge(merge(left, insertedRoot, this.#metrics), right, this.#metrics); + } + if (this.#root !== undefined) + this.#root.parent = undefined; + if (previousTailElement !== undefined) { + this.#tail = elementNode(previousTailElement); + } + this.#structureVersion += 1; + } + setDeleted(element, deleted) { + if (this.#hasLazyDeletions) + materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + const offset = elementOffset(element); + if (node === undefined || offset === undefined || element.deleted === deleted) + return; + this.#invalidateCausalView(); + updateNodeElementVisibility(node, offset, element, deleted, this.#metrics); + recomputeToRoot(node, this.#metrics, false); + } + deleteElement(element, deletedBy) { + if (this.#hasLazyDeletions) + materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + if (node === undefined) { + throw new Error("cannot delete an element outside the sequence"); + } + if (element.deleted && deletedBy === undefined) + return; + this.#invalidateCausalView(); + if (!element.deleted) { + updateNodeElementVisibility(node, elementOffset(element), element, true, this.#metrics); + recomputeToRoot(node, this.#metrics, false); + } + if (deletedBy === undefined) + return; + if (element.deletedByPeer === undefined && (element.deletedBy?.length ?? 0) === 0) { + element.deletedByPeer = deletedBy.peer; + element.deletedByCounter = deletedBy.counter; + } + else + (element.deletedBy ??= []).push(deletedBy); + this.#recordDeletionElement(element, deletedBy); + const previous = this.#maxCounterByPeer.get(deletedBy.peer) ?? 0; + if (deletedBy.counter + 1 > previous) { + this.#maxCounterByPeer.set(deletedBy.peer, deletedBy.counter + 1); + } + } + recordOperationId(id) { + const previous = this.#maxCounterByPeer.get(id.peer) ?? 0; + const next = Math.max(previous, id.counter + 1); + if (next === previous) + return; + this.#invalidateCausalView(); + this.#maxCounterByPeer.set(id.peer, next); + } + addDeletion(element, id) { + if (elementNode(element) === undefined) { + throw new Error("cannot delete an element outside the sequence"); + } + this.#invalidateCausalView(); + if (element.deletedByPeer === undefined && (element.deletedBy?.length ?? 0) === 0) { + element.deletedByPeer = id.peer; + element.deletedByCounter = id.counter; + } + else + (element.deletedBy ??= []).push(id); + this.#recordDeletionElement(element, id); + this.recordOperationId(id); + } + deleteIdSpan(startId, length, deletedBy) { + const spanLength = Math.abs(length); + if (spanLength === 0) + return; + if (spanLength === 1) { + const element = this.findById(startId); + if (element === undefined) + return; + this.deleteElement(element, deletedBy); + return; + } + const targetedLimit = Math.max(64, Math.min(512, this.allLength >>> 8)); + const deleteByTree = this.#root?.idRunCount === 1 || spanLength > targetedLimit; + const result = deleteByTree + ? markIdRunsDeleted(this.#root, singleCounterRangeRelation(startId.peer, startId.counter, startId.counter + spanLength), this.#metrics) + : this.#markLocatedIdSpanVisibility(startId, spanLength, true); + if (!result.found) + return; + if (deleteByTree) + this.#hasLazyDeletions = true; + this.#invalidateCausalView(); + if (deletedBy !== undefined) { + this.#deletionRuns.add(startId, spanLength, deletedBy, length >= 0 ? 1 : -1); + this.recordOperationId({ + peer: deletedBy.peer, + counter: deletedBy.counter + spanLength - 1, + }); + } + } + #markLocatedIdSpanVisibility(startId, length, deleted) { + const locations = this.#locationsByPeer.get(startId.peer); + if (locations === undefined) + return { found: false, changed: false }; + // Resolves locations to (node, offset) pairs so compact-span targets are + // read and written through column helpers without materializing views. + const targetNodes = []; + const targetOffsets = []; + locations.forEach(startId.counter, startId.counter + length, (location) => { + if (typeof location === "number") { + targetNodes.push(this.#nodeAtLocation(location)); + targetOffsets.push(location % SEQUENCE_LOCATION_STRIDE); + } + else { + targetNodes.push(elementNode(location)); + targetOffsets.push(elementOffset(location)); + } + }); + if (targetNodes.length === 0) + return { found: false, changed: false }; + if (targetNodes.length <= MAX_SEQUENCE_SPAN) { + const materializedNodes = []; + for (const node of targetNodes) { + if (materializedNodes.includes(node)) + continue; + materializedNodes.push(node); + materializeNodeDeletion(node, this.#metrics); + } + const changedNodes = []; + for (let index = 0; index < targetNodes.length; index += 1) { + const node = targetNodes[index]; + if (nodeDeleted(node, targetOffsets[index]) === deleted) + continue; + setNodeDeleted(node, targetOffsets[index], deleted); + if (!changedNodes.includes(node)) + changedNodes.push(node); + } + if (changedNodes.length === 0) + return { found: true, changed: false }; + for (const node of changedNodes) + recomputeOwn(node, this.#metrics); + for (const node of changedNodes) + recomputeToRoot(node, this.#metrics, false); + return { found: true, changed: true }; + } + const materializedNodes = new Set(); + for (const node of targetNodes) { + if (materializedNodes.has(node)) + continue; + materializedNodes.add(node); + materializeNodeDeletion(node, this.#metrics); + } + const changedNodes = new Set(); + for (let index = 0; index < targetNodes.length; index += 1) { + const node = targetNodes[index]; + if (nodeDeleted(node, targetOffsets[index]) === deleted) + continue; + setNodeDeleted(node, targetOffsets[index], deleted); + changedNodes.add(node); + } + if (changedNodes.size === 0) + return { found: true, changed: false }; + const affected = new Set(); + for (const node of changedNodes) { + recomputeOwn(node, this.#metrics); + let current = node; + while (current !== undefined) { + affected.add(current); + current = current.parent; + } + } + recomputeAffected(this.#root, affected, this.#metrics); + return { found: true, changed: true }; + } + setIdRunsDeleted(runs) { + if (runs.length === 0) + return; + const run = runs.length === 1 ? runs[0] : undefined; + const targetedLimit = Math.max(64, Math.min(512, this.allLength >>> 8)); + const updateByTree = run === undefined || this.#root?.idRunCount === 1 || run.length > targetedLimit; + const result = updateByTree + ? markIdRunsDeleted(this.#root, run === undefined + ? mapCounterRangeRelation(counterRangesByPeer(runs)) + : singleCounterRangeRelation(run.start.peer, run.start.counter, run.start.counter + run.length), this.#metrics) + : this.#markLocatedIdSpanVisibility(run.start, run.length, true); + if (!result.changed) + return; + if (updateByTree) + this.#hasLazyDeletions = true; + this.#invalidateCausalView(); + } + setIdRunsVisible(runs) { + if (runs.length === 0) + return; + const run = runs.length === 1 ? runs[0] : undefined; + const targetedLimit = Math.max(64, Math.min(512, this.allLength >>> 8)); + const updateByTree = run === undefined || this.#root?.idRunCount === 1 || run.length > targetedLimit; + const result = updateByTree + ? markIdRunsVisible(this.#root, run === undefined + ? mapCounterRangeRelation(counterRangesByPeer(runs)) + : singleCounterRangeRelation(run.start.peer, run.start.counter, run.start.counter + run.length), this.#metrics) + : this.#markLocatedIdSpanVisibility(run.start, run.length, false); + if (!result.changed) + return; + if (updateByTree) + this.#hasLazyDeletions = true; + this.#invalidateCausalView(); + } + canShowIdRunsAt(runs, version) { + for (const run of runs) { + if (run.start.counter + run.length > (version.get(run.start.peer) ?? 0)) { + return false; + } + } + if (this.#deletionRuns.hasDeletionAt(runs, version)) + return false; + for (const run of runs) { + if (this.#scalarDeletionTargetsByPeer + .get(run.start.peer) + ?.some(run.start.counter, run.start.counter + run.length, (location) => someSequenceDeletion(this.#elementAtDeletionLocation(location), (id) => id.counter < (version.get(id.peer) ?? 0)))) { + return false; + } + } + return true; + } + elementsDeletedBy(peer, start, end) { + const selected = new Set(); + for (const run of this.#deletionRuns.targetRunsDeletedBy(peer, start, end)) { + for (let offset = 0; offset < run.length; offset += 1) { + const location = this.#locationsByPeer + .get(run.start.peer) + ?.get(run.start.counter + offset); + if (location !== undefined) { + selected.add(typeof location === "number" + ? { + stableSequenceId: true, + peer: run.start.peer, + counter: run.start.counter + offset, + } + : location); + } + } + } + this.#deletionsByPeer.get(peer)?.forEach(start, end, (elementOrSet) => { + if (elementOrSet instanceof Set) { + for (const element of elementOrSet) + selected.add(element); + } + else { + selected.add(elementOrSet); + } + }); + const resolved = []; + for (const location of selected) { + const element = this.#elementAtDeletionLocation(location); + resolved.push({ element, index: this.physicalIndexOf(element) }); + } + resolved.sort((left, right) => left.index - right.index); + const output = []; + for (const { element } of resolved) + output.push(element); + return output; + } + idRunsDeletedBy(peer, start, end) { + const runs = this.#deletionRuns.targetRunsDeletedBy(peer, start, end); + this.#deletionsByPeer.get(peer)?.forEach(start, end, (elementOrSet) => { + if (elementOrSet instanceof Set) { + for (const location of elementOrSet) { + runs.push({ start: this.#elementAtDeletionLocation(location).id, length: 1 }); + } + } + else { + runs.push({ + start: this.#elementAtDeletionLocation(elementOrSet).id, + length: 1, + }); + } + }); + return normalizeSequenceIdRuns(runs); + } + someDeletion(element, predicate) { + return (this.#deletionRuns.someDeletion(element.id, predicate) || + someSequenceDeletion(element, predicate)); + } + everyDeletion(element, predicate) { + return (this.#deletionRuns.everyDeletion(element.id, predicate) && + everySequenceDeletion(element, predicate)); + } + moveVisible(from, to) { + const element = this.atVisible(from); + if (element === undefined || from === to) + return; + this.#invalidateCausalView(); + const physical = this.physicalIndexOf(element); + const [before, selectedAndAfter] = this.#split(this.#root, physical); + const [selected, after] = this.#split(selectedAndAfter, 1); + this.#root = merge(before, after, this.#metrics); + const destinationElement = this.atVisible(to); + const destination = destinationElement === undefined + ? this.allLength + : this.physicalIndexOf(destinationElement); + const [left, right] = this.#split(this.#root, destination); + this.#root = merge(merge(left, selected, this.#metrics), right, this.#metrics); + if (this.#root !== undefined) + this.#root.parent = undefined; + this.#tail = this.#root === undefined ? undefined : rightmost(this.#root); + this.#structureVersion += 1; + } + moveBefore(element, before) { + if (element === before) + return; + const physical = this.physicalIndexOf(element); + if (physical === undefined) + return; + this.#invalidateCausalView(); + const [left, selectedAndRight] = this.#split(this.#root, physical); + const [selected, right] = this.#split(selectedAndRight, 1); + this.#root = merge(left, right, this.#metrics); + const destination = before === undefined ? this.allLength : this.physicalIndexOf(before); + if (destination === undefined) { + this.#root = merge(this.#root, selected, this.#metrics); + } + else { + const [beforeDestination, afterDestination] = this.#split(this.#root, destination); + this.#root = merge(merge(beforeDestination, selected, this.#metrics), afterDestination, this.#metrics); + } + if (this.#root !== undefined) + this.#root.parent = undefined; + this.#tail = this.#root === undefined ? undefined : rightmost(this.#root); + this.#structureVersion += 1; + } + nextVisible(element) { + if (this.#hasLazyDeletions) + materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + const offset = elementOffset(element); + if (node === undefined || offset === undefined) + return undefined; + for (let index = offset + 1; index < nodeLength(node); index += 1) { + if (!nodeDeleted(node, index)) + return nodeElement(node, index); + } + const right = firstVisibleElement(node.right, this.#metrics); + if (right !== undefined) + return right; + let current = node; + while (current.parent !== undefined) { + const parent = current.parent; + if (current === parent.left) { + for (let index = 0; index < nodeLength(parent); index += 1) { + if (!nodeDeleted(parent, index)) + return nodeElement(parent, index); + } + const sibling = firstVisibleElement(parent.right, this.#metrics); + if (sibling !== undefined) + return sibling; + } + current = parent; + } + return undefined; + } + previousVisible(element) { + if (this.#hasLazyDeletions) + materializeElementDeletion(element, this.#metrics); + const node = elementNode(element); + const offset = elementOffset(element); + if (node === undefined || offset === undefined) + return undefined; + for (let index = offset - 1; index >= 0; index -= 1) { + if (!nodeDeleted(node, index)) + return nodeElement(node, index); + } + const left = lastVisibleElement(node.left, this.#metrics); + if (left !== undefined) + return left; + let current = node; + while (current.parent !== undefined) { + const parent = current.parent; + if (current === parent.right) { + for (let index = nodeLength(parent) - 1; index >= 0; index -= 1) { + if (!nodeDeleted(parent, index)) + return nodeElement(parent, index); + } + const sibling = lastVisibleElement(parent.left, this.#metrics); + if (sibling !== undefined) + return sibling; + } + current = parent; + } + return undefined; + } + isFullyIncluded(version) { + for (const [peer, counter] of this.#maxCounterByPeer) { + if ((version.get(peer) ?? 0) < counter) + return false; + } + return true; + } + causalVisible(version) { + if (this.isFullyIncluded(version)) + return this.visible(); + const view = this.causalView(version); + return view.range(0, view.length); + } + causalView(version) { + if (this.isFullyIncluded(version)) { + return { + length: this.visibleLength, + at: (index) => this.atVisible(index), + range: (start, end) => this.visibleRange(start, end), + idRuns: (start, end) => this.visibleIdRuns(start, end), + }; + } + const cachedIndex = this.#cachedCausalViews.findIndex((cached) => versionsEqual(cached.version, version)); + if (cachedIndex >= 0) { + const cached = this.#cachedCausalViews[cachedIndex]; + if (cachedIndex + 1 < this.#cachedCausalViews.length) { + this.#cachedCausalViews.splice(cachedIndex, 1); + this.#cachedCausalViews.push(cached); + } + return cached.view; + } + const causalVersion = new Map(version); + const included = (id) => id.counter < (causalVersion.get(id.peer) ?? 0); + const deletionIncluded = (deleteId) => !included(deleteId); + // Column-read visibility probe: scans stay allocation-free over compact + // spans and only returned elements are materialized by the callers. + const isVisible = (node, offset) => { + scratchSequenceCausalId.peer = nodePeer(node, offset); + scratchSequenceCausalId.counter = nodeCounter(node, offset); + return (included(scratchSequenceCausalId) && + this.#deletionRuns.everyDeletion(scratchSequenceCausalId, deletionIncluded) && + everyNodeElementDeletion(node, offset, deletionIncluded)); + }; + const causalCounts = new Map(); + const causalCount = (node) => { + if (node === undefined) + return 0; + const cached = causalCounts.get(node); + if (cached !== undefined) + return cached; + if (node.idRunCount === 1) { + const includedEnd = causalVersion.get(node.firstId.peer) ?? 0; + if (includedEnd <= node.firstId.counter) { + causalCounts.set(node, 0); + return 0; + } + if (includedEnd > node.lastId.counter && + this.canShowIdRunsAt([{ start: node.firstId, length: node.allCount }], causalVersion)) { + causalCounts.set(node, node.allCount); + return node.allCount; + } + } + let count = causalCount(node.left) + causalCount(node.right); + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (isVisible(node, offset)) + count += 1; + } + causalCounts.set(node, count); + return count; + }; + const length = causalCount(this.#root); + const view = { + length, + at: (index) => { + if (!Number.isSafeInteger(index) || index < 0 || index >= length) { + return undefined; + } + let node = this.#root; + let remaining = index; + while (node !== undefined) { + const leftCount = causalCount(node.left); + if (remaining < leftCount) { + node = node.left; + continue; + } + remaining -= leftCount; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (!isVisible(node, offset)) + continue; + if (remaining === 0) + return nodeElement(node, offset); + remaining -= 1; + } + node = node.right; + } + return undefined; + }, + range: (start, end) => { + const boundedStart = Math.max(0, Math.min(start, length)); + const boundedEnd = Math.max(boundedStart, Math.min(end, length)); + const output = []; + collectCausalRange(this.#root, boundedStart, boundedEnd, output, causalCount, isVisible, this.#metrics); + return output; + }, + idRuns: (start, end) => { + const boundedStart = Math.max(0, Math.min(start, length)); + const boundedEnd = Math.max(boundedStart, Math.min(end, length)); + const output = []; + collectCausalIdRuns(this.#root, boundedStart, boundedEnd, output, causalCount, isVisible); + return output; + }, + }; + this.#cachedCausalViews.push({ version: causalVersion, view }); + if (this.#cachedCausalViews.length > MAX_CACHED_CAUSAL_VIEWS) { + this.#cachedCausalViews.shift(); + } + return view; + } + #invalidateCausalView() { + this.#cachedCausalViews.length = 0; + } + reset() { + this.#invalidateCausalView(); + this.#root = undefined; + this.#tail = undefined; + this.#locationsByPeer.clear(); + this.#locationsByLamportPeer.clear(); + this.#nodesByLocationId.length = 0; + this.#deletionsByPeer.clear(); + this.#scalarDeletionTargetsByPeer.clear(); + this.#deletionRuns.reset(); + this.#maxCounterByPeer.clear(); + this.#hasLazyDeletions = false; + this.#structureVersion += 1; + } + reservePeerCounters(peer, endExclusive) { + if (!Number.isSafeInteger(endExclusive) || endExclusive < 0) { + throw new RangeError("sequence peer counter capacity is out of range"); + } + let locations = this.#locationsByPeer.get(peer); + if (locations === undefined) { + locations = new CounterStore(); + this.#locationsByPeer.set(peer, locations); + } + locations.reserveDense(endExclusive); + } + /** + * Loads globally validated spans into an empty index in physical order. + * The Cartesian-tree build preserves the same priority treap as repeated + * tail insertion while recomputing every node only once. + */ + loadValidatedSpans(spans) { + if (this.#root !== undefined || this.#nodesByLocationId.length !== 0) { + throw new Error("validated sequence spans require an empty index"); + } + if (spans.length === 0) + return; + this.#invalidateCausalView(); + const stack = []; + let root; + let tail; + for (const span of spans) { + const node = this.#newNode(span, true); + tail = node; + let left; + while (stack.length > 0 && stack[stack.length - 1].priority > node.priority) { + left = stack.pop(); + } + node.left = left; + const parent = stack[stack.length - 1]; + if (parent === undefined) { + root = node; + } + else { + parent.right = node; + } + stack.push(node); + } + const pending = [root]; + const postorder = []; + while (pending.length > 0) { + const node = pending.pop(); + postorder.push(node); + if (node.left !== undefined) + pending.push(node.left); + if (node.right !== undefined) + pending.push(node.right); + } + for (let index = postorder.length - 1; index >= 0; index -= 1) { + recompute(postorder[index], this.#metrics); + } + root.parent = undefined; + this.#root = root; + this.#tail = tail; + this.#structureVersion += 1; + } + #newNode(storage, recordNew) { + const element = Array.isArray(storage) + ? storage.length === 1 + ? storage[0] + : [...storage] + : storage; + const isSpan = element instanceof SequenceSpan; + const length = isSpan + ? element.length + : Array.isArray(element) + ? element.length + : 1; + if (length === 0 || length > MAX_SEQUENCE_SPAN) { + throw new RangeError("sequence span length is out of range"); + } + this.#randomState ^= this.#randomState << 13; + this.#randomState ^= this.#randomState >>> 17; + this.#randomState ^= this.#randomState << 5; + // Keep priorities inside the compressed-Smi range [0, 2^30) so every node + // avoids boxing its priority; the treap only needs a deterministic total + // order, and 30 bits of entropy is ample for its node counts. + const priority = this.#randomState >>> 2; + const locationId = this.#nodesByLocationId.length; + const node = { + element, + isSpan, + ownCount: length, + locationId, + priority, + left: undefined, + right: undefined, + parent: undefined, + ownVisibleCount: 0, + ownVisibleUtf16: 0, + ownVisibleUtf8: 0, + ownFirstVisibleId: undefined, + ownLastVisibleId: undefined, + ownVisibleIdRunCount: 0, + ownFirstId: { peer: 0n, counter: 0 }, + ownLastId: { peer: 0n, counter: 0 }, + ownIdRunCount: 0, + ownUtf16: 0, + ownUtf8: 0, + visibleOffsets: undefined, + visibleUtf16Prefix: undefined, + visibleUtf8Prefix: undefined, + allCount: 0, + visibleCount: 0, + visibleUtf16: 0, + visibleUtf8: 0, + firstVisibleId: undefined, + lastVisibleId: undefined, + visibleIdRunCount: 0, + firstId: { peer: 0n, counter: 0 }, + lastId: { peer: 0n, counter: 0 }, + idRunCount: 0, + allUtf16: 0, + allUtf8: 0, + lazyDeleted: false, + lazyVisible: false, + }; + this.#nodesByLocationId.push(node); + recomputeOwn(node, this.#metrics); + recompute(node, this.#metrics); + if (recordNew) + this.#recordNewElements(node); + else + this.#assignLocations(node); + return node; + } + #recordNewElements(node) { + for (let offset = 0; offset < nodeLength(node); offset += 1) { + this.#recordNewElement(node, offset); + } + } + #recordNewElement(node, offset) { + if (!node.isSpan) { + this.#recordNewPlainElement(node, offset, nodeElement(node, offset)); + return; + } + const idPeer = nodePeer(node, offset); + const idCounter = nodeCounter(node, offset); + const location = node.locationId * SEQUENCE_LOCATION_STRIDE + offset; + let peer = this.#locationsByPeer.get(idPeer); + if (peer === undefined) { + peer = new CounterStore(); + this.#locationsByPeer.set(idPeer, peer); + } + peer.set(idCounter, location); + const lamport = nodeLamport(node, offset); + if (lamport !== undefined) { + this.#locationsByLamportPeer.get(idPeer)?.set(lamport, location); + } + const operationEnd = idCounter + 1; + const previousOperationEnd = this.#maxCounterByPeer.get(idPeer) ?? 0; + if (operationEnd > previousOperationEnd) { + this.#maxCounterByPeer.set(idPeer, operationEnd); + } + const deletionIds = node.element.deletionIdsAt(offset); + if (deletionIds.length > 0) { + const element = nodeElement(node, offset); + for (const deletionId of deletionIds) { + this.#recordDeletionElement(element, deletionId); + this.recordOperationId(deletionId); + } + } + } + #recordNewPlainElement(node, offset, element) { + setElementLocation(element, node, offset); + let peer = this.#locationsByPeer.get(element.id.peer); + if (peer === undefined) { + peer = new CounterStore(); + this.#locationsByPeer.set(element.id.peer, peer); + } + peer.set(element.id.counter, element); + if (element.lamport !== undefined) { + this.#locationsByLamportPeer.get(element.id.peer)?.set(element.lamport, element); + } + const operationEnd = element.id.counter + 1; + const previousOperationEnd = this.#maxCounterByPeer.get(element.id.peer) ?? 0; + if (operationEnd > previousOperationEnd) { + this.#maxCounterByPeer.set(element.id.peer, operationEnd); + } + // Iterates deletion metadata in place: the common fresh element carries + // none and must not allocate an empty id list. + if (element.deletedByPeer !== undefined && element.deletedByCounter !== undefined) { + const deletionId = { + peer: element.deletedByPeer, + counter: element.deletedByCounter, + }; + this.#recordDeletionElement(element, deletionId); + this.recordOperationId(deletionId); + } + const deletedBy = element.deletedBy; + if (deletedBy !== undefined) { + for (let index = 0; index < deletedBy.length; index += 1) { + const deletionId = deletedBy[index]; + this.#recordDeletionElement(element, deletionId); + this.recordOperationId(deletionId); + } + } + } + #assignLocations(node) { + if (node.isSpan) { + node.element.forEachRetained((element, offset) => { + setElementLocation(element, node, offset); + }); + } + else { + for (let offset = 0; offset < nodeLength(node); offset += 1) { + const element = nodeElement(node, offset); + if (elementNode(element) === undefined) { + throw new Error("cannot relocate an unindexed sequence element"); + } + setElementLocation(element, node, offset); + } + } + for (let offset = 0; offset < nodeLength(node); offset += 1) { + const peer = nodePeer(node, offset); + const counter = nodeCounter(node, offset); + const location = node.isSpan + ? node.locationId * SEQUENCE_LOCATION_STRIDE + offset + : nodeElement(node, offset); + this.#locationsByPeer.get(peer)?.set(counter, location); + const lamport = nodeLamport(node, offset); + if (lamport !== undefined) { + this.#locationsByLamportPeer.get(peer)?.set(lamport, location); + } + } + } + #nodeAtLocation(location) { + // location is non-negative and below 2^31; SEQUENCE_LOCATION_STRIDE is 64. + const node = this.#nodesByLocationId[(location / SEQUENCE_LOCATION_STRIDE) | 0]; + if (node === undefined) + throw new Error("sequence location points to a missing node"); + return node; + } + #elementAtLocation(location) { + return nodeElement(this.#nodeAtLocation(location), location % SEQUENCE_LOCATION_STRIDE); + } + #elementAtDeletionLocation(location) { + if ("stableSequenceId" in location) { + const element = this.findById(location); + if (element === undefined) + throw new Error("deleted sequence element is missing"); + return element; + } + return location; + } + #recordDeletionElement(element, id) { + const node = elementNode(element); + const offset = elementOffset(element); + const location = node?.isSpan === true && offset !== undefined + ? { + stableSequenceId: true, + peer: element.id.peer, + counter: element.id.counter, + } + : element; + let targets = this.#scalarDeletionTargetsByPeer.get(element.id.peer); + if (targets === undefined) { + targets = new PagedCounterStore(); + this.#scalarDeletionTargetsByPeer.set(element.id.peer, targets); + } + targets.set(element.id.counter, location); + let counters = this.#deletionsByPeer.get(id.peer); + if (counters === undefined) { + counters = new CounterStore(); + this.#deletionsByPeer.set(id.peer, counters); + } + const elementOrSet = counters.get(id.counter); + if (elementOrSet === undefined) { + counters.set(id.counter, location); + } + else if (elementOrSet instanceof Set) { + elementOrSet.add(location); + } + else if (elementOrSet !== location) { + counters.set(id.counter, new Set([elementOrSet, location])); + } + } + #split(root, count) { + if (root === undefined) + return [undefined, undefined]; + // Iterative split: descend to the cut node, then stitch the two result + // trees on the way back up through the original parent links. This + // mirrors the recursive unwind without a per-level [left, right] tuple. + let cut = root; + let last; + let lastWentLeft = false; + let remaining = count; + let cutLeftCount = 0; + let cutOwnEnd = 0; + while (cut !== undefined) { + last = cut; + pushNodeDeletion(cut, this.#metrics); + const leftCount = allCount(cut.left); + const ownEnd = leftCount + nodeLength(cut); + if (remaining < leftCount) { + lastWentLeft = true; + cut = cut.left; + } + else if (remaining > ownEnd) { + lastWentLeft = false; + remaining -= ownEnd; + cut = cut.right; + } + else { + cutLeftCount = leftCount; + cutOwnEnd = ownEnd; + break; + } + } + let leftResult; + let rightResult; + let child; + const cutParent = (cut !== undefined ? cut : last).parent; + if (cut !== undefined) { + if (remaining === cutLeftCount) { + leftResult = cut.left; + cut.left = undefined; + recompute(cut, this.#metrics); + cut.parent = undefined; + if (leftResult !== undefined) + leftResult.parent = undefined; + rightResult = cut; + } + else if (remaining === cutOwnEnd) { + rightResult = cut.right; + cut.right = undefined; + recompute(cut, this.#metrics); + cut.parent = undefined; + if (rightResult !== undefined) + rightResult.parent = undefined; + leftResult = cut; + } + else { + const ownOffset = remaining - cutLeftCount; + const storage = cut.element; + const oldRight = cut.right; + cut.element = sliceNodeStorage(storage, 0, ownOffset); + cut.ownCount = ownOffset; + cut.right = undefined; + recomputeOwn(cut, this.#metrics); + recompute(cut, this.#metrics); + cut.parent = undefined; + this.#assignLocations(cut); + const rightRoot = this.#newNode(sliceNodeStorage(storage, ownOffset, nodeLengthFromStorage(storage)), false); + rightResult = merge(rightRoot, oldRight, this.#metrics); + if (rightResult !== undefined) + rightResult.parent = undefined; + leftResult = cut; + } + child = cut; + } + else { + // `count` fell outside the tree: the deepest frame reattaches nothing + // and folds the last visited node into the matching result. + child = last; + if (lastWentLeft) { + child.left = undefined; + recompute(child, this.#metrics); + child.parent = undefined; + rightResult = child; + } + else { + child.right = undefined; + recompute(child, this.#metrics); + child.parent = undefined; + leftResult = child; + } + } + let up = cutParent; + while (child !== root && up !== undefined) { + const nextUp = up.parent; + if (up.left === child) { + up.left = rightResult; + recompute(up, this.#metrics); + if (leftResult !== undefined) + leftResult.parent = undefined; + up.parent = undefined; + rightResult = up; + } + else { + up.right = leftResult; + recompute(up, this.#metrics); + if (rightResult !== undefined) + rightResult.parent = undefined; + up.parent = undefined; + leftResult = up; + } + child = up; + up = nextUp; + } + return [leftResult, rightResult]; + } + #appendToPreviousNode(position, elements) { + if (position === 0) + return false; + const previous = this.atPhysical(position - 1); + if (previous === undefined) + return false; + const node = elementNode(previous); + const offset = elementOffset(previous); + const length = nodeLength(node); + if (node.isSpan || + offset !== length - 1 || + length + elements.length > MAX_SEQUENCE_SPAN) { + return false; + } + const appendingAtTail = position === this.allLength; + const storage = node.element; + const values = Array.isArray(storage) ? storage : [storage]; + node.element = values; + const visibleOffsets = node.visibleOffsets ?? [0, node.ownVisibleCount]; + const visibleUtf16Prefix = node.visibleUtf16Prefix ?? [0, node.ownVisibleUtf16]; + const visibleUtf8Prefix = node.visibleUtf8Prefix ?? [0, node.ownVisibleUtf8]; + for (let offset = 0; offset < elements.length; offset += 1) { + const element = elements[offset]; + values.push(element); + this.#recordNewElement(node, length + offset); + if (!sequenceIdsContinue(node.ownLastId, element.id)) { + node.ownIdRunCount += 1; + } + node.ownLastId = element.id; + const metrics = this.#metrics(element); + node.ownUtf16 += metrics.utf16; + node.ownUtf8 += metrics.utf8; + if (!element.deleted) { + if (node.ownVisibleCount === 0) { + node.ownFirstVisibleId = element.id; + node.ownVisibleIdRunCount = 1; + } + else if (!sequenceIdsContinue(node.ownLastVisibleId, element.id)) { + node.ownVisibleIdRunCount += 1; + } + node.ownLastVisibleId = element.id; + node.ownVisibleCount += 1; + node.ownVisibleUtf16 += metrics.utf16; + node.ownVisibleUtf8 += metrics.utf8; + } + visibleOffsets.push(node.ownVisibleCount); + visibleUtf16Prefix.push(node.ownVisibleUtf16); + visibleUtf8Prefix.push(node.ownVisibleUtf8); + } + node.ownCount += elements.length; + node.visibleOffsets = visibleOffsets; + node.visibleUtf16Prefix = visibleUtf16Prefix; + node.visibleUtf8Prefix = visibleUtf8Prefix; + recomputeToRoot(node, this.#metrics); + if (appendingAtTail) + this.#tail = node; + return true; + } + #appendSpanToPreviousNode(position, span) { + if (position === 0) + return false; + const previous = this.atPhysical(position - 1); + if (previous === undefined) + return false; + const node = elementNode(previous); + const offset = elementOffset(previous); + const length = nodeLength(node); + if (!node.isSpan || + offset !== length - 1 || + length + span.length > MAX_SEQUENCE_SPAN || + !node.element.append(span)) { + return false; + } + node.ownCount += span.length; + const appendingAtTail = position === this.allLength; + const visibleOffsets = node.visibleOffsets ?? [0, node.ownVisibleCount]; + const visibleUtf16Prefix = node.visibleUtf16Prefix ?? [0, node.ownVisibleUtf16]; + const visibleUtf8Prefix = node.visibleUtf8Prefix ?? [0, node.ownVisibleUtf8]; + let lastPeer = node.ownLastId.peer; + let lastCounter = node.ownLastId.counter; + let lastVisiblePeer = node.ownLastVisibleId?.peer; + let lastVisibleCounter = node.ownLastVisibleId?.counter ?? 0; + for (let insertedOffset = 0; insertedOffset < span.length; insertedOffset += 1) { + const offsetInNode = length + insertedOffset; + this.#recordNewElement(node, offsetInNode); + const peer = nodePeer(node, offsetInNode); + const counter = nodeCounter(node, offsetInNode); + if (!sequenceIdScalarsContinue(lastPeer, lastCounter, peer, counter)) { + node.ownIdRunCount += 1; + } + lastPeer = peer; + lastCounter = counter; + const value = nodeMetrics(node, offsetInNode, this.#metrics); + node.ownUtf16 += value.utf16; + node.ownUtf8 += value.utf8; + if (!nodeDeleted(node, offsetInNode)) { + if (node.ownVisibleCount === 0) { + node.ownFirstVisibleId = { peer, counter }; + node.ownVisibleIdRunCount = 1; + } + else if (!sequenceIdScalarsContinue(lastVisiblePeer, lastVisibleCounter, peer, counter)) { + node.ownVisibleIdRunCount += 1; + } + lastVisiblePeer = peer; + lastVisibleCounter = counter; + node.ownVisibleCount += 1; + node.ownVisibleUtf16 += value.utf16; + node.ownVisibleUtf8 += value.utf8; + } + visibleOffsets.push(node.ownVisibleCount); + visibleUtf16Prefix.push(node.ownVisibleUtf16); + visibleUtf8Prefix.push(node.ownVisibleUtf8); + } + node.ownLastId = { peer: lastPeer, counter: lastCounter }; + node.ownLastVisibleId = + lastVisiblePeer === undefined + ? undefined + : { peer: lastVisiblePeer, counter: lastVisibleCounter }; + node.visibleOffsets = visibleOffsets; + node.visibleUtf16Prefix = visibleUtf16Prefix; + node.visibleUtf8Prefix = visibleUtf8Prefix; + recomputeToRoot(node, this.#metrics); + if (appendingAtTail) + this.#tail = node; + return true; + } + #insertSingleNode(root, position, inserted) { + if (root === undefined) { + inserted.parent = undefined; + return inserted; + } + pushNodeDeletion(root, this.#metrics); + if (inserted.priority < root.priority) { + const [left, right] = this.#split(root, position); + inserted.left = left; + inserted.right = right; + recompute(inserted, this.#metrics); + inserted.parent = undefined; + return inserted; + } + const leftCount = allCount(root.left); + const ownEnd = leftCount + nodeLength(root); + if (position <= leftCount) { + root.left = this.#insertSingleNode(root.left, position, inserted); + recompute(root, this.#metrics); + root.parent = undefined; + return root; + } + if (position >= ownEnd) { + root.right = this.#insertSingleNode(root.right, position - ownEnd, inserted); + recompute(root, this.#metrics); + root.parent = undefined; + return root; + } + const [left, right] = this.#split(root, position); + return merge(merge(left, inserted, this.#metrics), right, this.#metrics); + } + #appendAtTail(left, right) { + if (left === undefined || right === undefined) { + if (right !== undefined) + this.#tail = rightmost(right); + return merge(left, right, this.#metrics); + } + this.#tail = rightmost(right); + return merge(left, right, this.#metrics); + } +} +// Reused deletion id passed to predicates so each visited deleted element +// does not allocate a short-lived id object. Predicates must not retain the +// id argument or query deletions reentrantly. +const scratchSequenceDeletionId = { peer: 0n, counter: 0 }; +// Reused id targets for visible-id-run starts and causal-view visibility +// probes, so column scans do not allocate an id per visited element. +// `appendSequenceIdRun` copies the run start before retaining it; deletion +// predicates must not retain the ids they receive. +const scratchSequenceRunId = { peer: 0n, counter: 0 }; +const scratchSequenceCausalId = { peer: 0n, counter: 0 }; +export function someSequenceDeletion(element, predicate) { + if (element.deletedByPeer !== undefined && element.deletedByCounter !== undefined) { + scratchSequenceDeletionId.peer = element.deletedByPeer; + scratchSequenceDeletionId.counter = element.deletedByCounter; + if (predicate(scratchSequenceDeletionId)) + return true; + } + return element.deletedBy?.some(predicate) ?? false; +} +export function everySequenceDeletion(element, predicate) { + if (element.deletedByPeer !== undefined && element.deletedByCounter !== undefined) { + scratchSequenceDeletionId.peer = element.deletedByPeer; + scratchSequenceDeletionId.counter = element.deletedByCounter; + if (!predicate(scratchSequenceDeletionId)) + return false; + } + return element.deletedBy?.every(predicate) ?? true; +} +/** + * Column-read variant of `everySequenceDeletion`: compact-span deletion + * metadata is checked through `deletionIdsAt` without materializing an + * element view. Plain elements are read in place. + */ +function everyNodeElementDeletion(node, offset, predicate) { + if (node.isSpan) { + const deletionIds = node.element.deletionIdsAt(offset); + for (let index = 0; index < deletionIds.length; index += 1) { + if (!predicate(deletionIds[index])) + return false; + } + return true; + } + const storage = node.element; + return everySequenceDeletion(Array.isArray(storage) ? storage[offset] : storage, predicate); +} +function elementNode(element) { + return element[SEQUENCE_NODE]; +} +function elementOffset(element) { + return element[SEQUENCE_OFFSET]; +} +function setElementLocation(element, node, offset) { + const located = element; + located[SEQUENCE_NODE] = node; + located[SEQUENCE_OFFSET] = offset; +} +function nodeLength(node) { + return node.ownCount; +} +function nodeLengthFromStorage(storage) { + return storage instanceof SequenceSpan + ? storage.length + : Array.isArray(storage) + ? storage.length + : 1; +} +function nodeElement(node, offset) { + if (node.isSpan) { + const element = node.element.elementAt(offset); + setElementLocation(element, node, offset); + return element; + } + const storage = node.element; + return Array.isArray(storage) ? storage[offset] : storage; +} +function nodePeer(node, offset) { + return node.isSpan + ? node.element.peerAt(offset) + : Array.isArray(node.element) + ? node.element[offset].id.peer + : node.element.id.peer; +} +function nodeCounter(node, offset) { + return node.isSpan + ? node.element.counterAt(offset) + : Array.isArray(node.element) + ? node.element[offset].id.counter + : node.element.id.counter; +} +function findNextIncludedPhysical(node, nodeStart, start, version) { + if (node === undefined || nodeStart + node.allCount <= start) + return undefined; + if (node.idRunCount === 1 && + (version.get(node.firstId.peer) ?? 0) <= node.firstId.counter) { + return undefined; + } + const leftCount = allCount(node.left); + const ownStart = nodeStart + leftCount; + const fromLeft = findNextIncludedPhysical(node.left, nodeStart, start, version); + if (fromLeft !== undefined) + return fromLeft; + const offsetStart = Math.max(0, start - ownStart); + if (offsetStart < nodeLength(node)) { + if (node.ownIdRunCount === 1) { + const peer = nodePeer(node, offsetStart); + const counter = nodeCounter(node, offsetStart); + if (counter < (version.get(peer) ?? 0)) { + return { + element: nodeElement(node, offsetStart), + index: ownStart + offsetStart, + }; + } + } + else { + for (let offset = offsetStart; offset < nodeLength(node); offset += 1) { + const peer = nodePeer(node, offset); + const counter = nodeCounter(node, offset); + if (counter < (version.get(peer) ?? 0)) { + return { element: nodeElement(node, offset), index: ownStart + offset }; + } + } + } + } + return findNextIncludedPhysical(node.right, ownStart + nodeLength(node), start, version); +} +function nodeLamport(node, offset) { + return node.isSpan + ? node.element.lamportAt(offset) + : Array.isArray(node.element) + ? node.element[offset].lamport + : node.element.lamport; +} +function nodeDeleted(node, offset) { + return node.isSpan + ? node.element.deletedAt(offset) + : Array.isArray(node.element) + ? node.element[offset].deleted + : node.element.deleted; +} +function setNodeDeleted(node, offset, deleted) { + if (node.isSpan) { + node.element.setDeletedAt(offset, deleted); + } + else if (Array.isArray(node.element)) { + node.element[offset].deleted = deleted; + } + else { + node.element.deleted = deleted; + } +} +function nodeMetrics(node, offset, metrics) { + return node.isSpan + ? node.element.metricsAt(offset) + : metrics(Array.isArray(node.element) + ? node.element[offset] + : node.element); +} +function sliceNodeStorage(storage, start, end) { + if (storage instanceof SequenceSpan) + return storage.slice(start, end); + const elements = Array.isArray(storage) ? storage.slice(start, end) : [storage]; + return elements.length === 1 ? elements[0] : elements; +} +function appendNodeElements(node, output) { + if (node.isSpan) { + const span = node.element; + for (let offset = 0; offset < span.length; offset += 1) { + output.push(nodeElement(node, offset)); + } + } + else if (Array.isArray(node.element)) + output.push(...node.element); + else + output.push(node.element); +} +function allCount(node) { + return node?.allCount ?? 0; +} +function visibleCount(node) { + return node?.visibleCount ?? 0; +} +function visibleMetric(node, metric) { + if (node === undefined) + return 0; + return metric === "utf16" ? node.visibleUtf16 : node.visibleUtf8; +} +function ownVisibleCount(node) { + return node.ownVisibleCount; +} +function visibleElementsBefore(node, offset) { + if (node.visibleOffsets !== undefined) + return node.visibleOffsets[offset]; + let count = 0; + for (let index = 0; index < offset; index += 1) { + if (!nodeDeleted(node, index)) + count += 1; + } + return count; +} +function ownVisibleMetric(node, metric, _metrics) { + return metric === "utf16" ? node.ownVisibleUtf16 : node.ownVisibleUtf8; +} +function visibleMetricBefore(node, offset, metric, metrics) { + const prefix = metric === "utf16" ? node.visibleUtf16Prefix : node.visibleUtf8Prefix; + if (prefix !== undefined) + return prefix[offset]; + let total = 0; + for (let index = 0; index < offset; index += 1) { + if (!nodeDeleted(node, index)) + total += nodeMetrics(node, index, metrics)[metric]; + } + return total; +} +function physicalOffsetAtVisibleIndex(node, index) { + if (node.ownVisibleCount === node.ownCount) + return index; + const prefix = node.visibleOffsets; + if (prefix === undefined) + return 0; + let low = 1; + let high = prefix.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (prefix[middle] > index) + high = middle; + else + low = middle + 1; + } + return low - 1; +} +function physicalOffsetAtMetricOffset(node, offset, metric) { + const prefix = metric === "utf16" ? node.visibleUtf16Prefix : node.visibleUtf8Prefix; + if (prefix === undefined) { + if (offset === 0) + return 0; + return offset === ownVisibleMetric(node, metric, listMetrics) ? 1 : undefined; + } + let low = 0; + let high = prefix.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (prefix[middle] >= offset) + high = middle; + else + low = middle + 1; + } + return prefix[low] === offset ? low : undefined; +} +function updateNodeElementVisibility(node, offset, element, deleted, metrics) { + const value = metrics(element); + const direction = deleted ? -1 : 1; + setNodeDeleted(node, offset, deleted); + node.ownVisibleCount += direction; + node.ownVisibleUtf16 += direction * value.utf16; + node.ownVisibleUtf8 += direction * value.utf8; + recomputeOwnIdRuns(node); + if (nodeLength(node) === 1) + return; + for (let index = offset + 1; index < node.visibleOffsets.length; index += 1) { + node.visibleOffsets[index] += direction; + node.visibleUtf16Prefix[index] += direction * value.utf16; + node.visibleUtf8Prefix[index] += direction * value.utf8; + } +} +function recomputeOwn(node, metrics) { + if (!node.isSpan) { + recomputeOwnElements(node, metrics); + return; + } + const length = nodeLength(node); + const firstPeer = nodePeer(node, 0); + const firstCounter = nodeCounter(node, 0); + node.ownFirstId = { peer: firstPeer, counter: firstCounter }; + node.ownLastId = + length === 1 + ? node.ownFirstId + : { + peer: nodePeer(node, length - 1), + counter: nodeCounter(node, length - 1), + }; + node.ownIdRunCount = 1; + let previousPeer = firstPeer; + let previousCounter = firstCounter; + for (let offset = 1; offset < length; offset += 1) { + const peer = nodePeer(node, offset); + const counter = nodeCounter(node, offset); + if (!sequenceIdScalarsContinue(previousPeer, previousCounter, peer, counter)) { + node.ownIdRunCount += 1; + } + previousPeer = peer; + previousCounter = counter; + } + if (length === 1) { + const value = nodeMetrics(node, 0, metrics); + node.ownUtf16 = value.utf16; + node.ownUtf8 = value.utf8; + node.visibleOffsets = undefined; + node.visibleUtf16Prefix = undefined; + node.visibleUtf8Prefix = undefined; + if (nodeDeleted(node, 0)) { + node.ownVisibleCount = 0; + node.ownVisibleUtf16 = 0; + node.ownVisibleUtf8 = 0; + node.ownFirstVisibleId = undefined; + node.ownLastVisibleId = undefined; + node.ownVisibleIdRunCount = 0; + } + else { + node.ownVisibleCount = 1; + node.ownVisibleUtf16 = value.utf16; + node.ownVisibleUtf8 = value.utf8; + node.ownFirstVisibleId = node.ownFirstId; + node.ownLastVisibleId = node.ownFirstId; + node.ownVisibleIdRunCount = 1; + } + return; + } + node.ownVisibleCount = 0; + node.ownVisibleUtf16 = 0; + node.ownVisibleUtf8 = 0; + node.ownUtf16 = 0; + node.ownUtf8 = 0; + node.ownFirstVisibleId = undefined; + node.ownLastVisibleId = undefined; + node.ownVisibleIdRunCount = 0; + const visibleOffsets = [0]; + const visibleUtf16Prefix = [0]; + const visibleUtf8Prefix = [0]; + let lastVisiblePeer; + let lastVisibleCounter = 0; + for (let offset = 0; offset < length; offset += 1) { + const peer = nodePeer(node, offset); + const counter = nodeCounter(node, offset); + const value = nodeMetrics(node, offset, metrics); + node.ownUtf16 += value.utf16; + node.ownUtf8 += value.utf8; + if (!nodeDeleted(node, offset)) { + if (node.ownVisibleCount === 0) { + node.ownFirstVisibleId = { peer, counter }; + node.ownVisibleIdRunCount = 1; + } + else if (!sequenceIdScalarsContinue(lastVisiblePeer, lastVisibleCounter, peer, counter)) { + node.ownVisibleIdRunCount += 1; + } + lastVisiblePeer = peer; + lastVisibleCounter = counter; + node.ownVisibleCount += 1; + node.ownVisibleUtf16 += value.utf16; + node.ownVisibleUtf8 += value.utf8; + } + visibleOffsets.push(node.ownVisibleCount); + visibleUtf16Prefix.push(node.ownVisibleUtf16); + visibleUtf8Prefix.push(node.ownVisibleUtf8); + } + node.ownLastVisibleId = + lastVisiblePeer === undefined + ? undefined + : { peer: lastVisiblePeer, counter: lastVisibleCounter }; + node.visibleOffsets = visibleOffsets; + node.visibleUtf16Prefix = visibleUtf16Prefix; + node.visibleUtf8Prefix = visibleUtf8Prefix; +} +function recomputeOwnElements(node, metrics) { + const storage = node.element; + const elements = Array.isArray(storage) ? storage : undefined; + const first = elements?.[0] ?? storage; + const last = elements?.at(-1) ?? storage; + node.ownFirstId = first.id; + node.ownLastId = last.id; + node.ownIdRunCount = 1; + if (elements !== undefined) { + for (let offset = 1; offset < elements.length; offset += 1) { + if (!sequenceIdsContinue(elements[offset - 1].id, elements[offset].id)) { + node.ownIdRunCount += 1; + } + } + } + if (elements === undefined) { + const value = metrics(storage); + node.ownUtf16 = value.utf16; + node.ownUtf8 = value.utf8; + node.visibleOffsets = undefined; + node.visibleUtf16Prefix = undefined; + node.visibleUtf8Prefix = undefined; + if (storage.deleted) { + node.ownVisibleCount = 0; + node.ownVisibleUtf16 = 0; + node.ownVisibleUtf8 = 0; + node.ownFirstVisibleId = undefined; + node.ownLastVisibleId = undefined; + node.ownVisibleIdRunCount = 0; + } + else { + node.ownVisibleCount = 1; + node.ownVisibleUtf16 = value.utf16; + node.ownVisibleUtf8 = value.utf8; + node.ownFirstVisibleId = storage.id; + node.ownLastVisibleId = storage.id; + node.ownVisibleIdRunCount = 1; + } + return; + } + node.ownVisibleCount = 0; + node.ownVisibleUtf16 = 0; + node.ownVisibleUtf8 = 0; + node.ownUtf16 = 0; + node.ownUtf8 = 0; + node.ownFirstVisibleId = undefined; + node.ownLastVisibleId = undefined; + node.ownVisibleIdRunCount = 0; + const visibleOffsets = [0]; + const visibleUtf16Prefix = [0]; + const visibleUtf8Prefix = [0]; + for (const element of elements) { + const value = metrics(element); + node.ownUtf16 += value.utf16; + node.ownUtf8 += value.utf8; + if (!element.deleted) { + if (node.ownVisibleCount === 0) { + node.ownFirstVisibleId = element.id; + node.ownVisibleIdRunCount = 1; + } + else if (!sequenceIdsContinue(node.ownLastVisibleId, element.id)) { + node.ownVisibleIdRunCount += 1; + } + node.ownLastVisibleId = element.id; + node.ownVisibleCount += 1; + node.ownVisibleUtf16 += value.utf16; + node.ownVisibleUtf8 += value.utf8; + } + visibleOffsets.push(node.ownVisibleCount); + visibleUtf16Prefix.push(node.ownVisibleUtf16); + visibleUtf8Prefix.push(node.ownVisibleUtf8); + } + node.visibleOffsets = visibleOffsets; + node.visibleUtf16Prefix = visibleUtf16Prefix; + node.visibleUtf8Prefix = visibleUtf8Prefix; +} +function recomputeVisibility(node) { + const left = node.left; + const right = node.right; + node.visibleCount = + (left?.visibleCount ?? 0) + node.ownVisibleCount + (right?.visibleCount ?? 0); + node.visibleUtf16 = + (left?.visibleUtf16 ?? 0) + node.ownVisibleUtf16 + (right?.visibleUtf16 ?? 0); + node.visibleUtf8 = + (left?.visibleUtf8 ?? 0) + node.ownVisibleUtf8 + (right?.visibleUtf8 ?? 0); + node.firstVisibleId = + left?.firstVisibleId ?? node.ownFirstVisibleId ?? right?.firstVisibleId; + node.lastVisibleId = + right?.lastVisibleId ?? node.ownLastVisibleId ?? left?.lastVisibleId; + node.visibleIdRunCount = + (left?.visibleIdRunCount ?? 0) + + node.ownVisibleIdRunCount + + (right?.visibleIdRunCount ?? 0); + const leftLastVisible = left?.lastVisibleId; + const ownFirstVisible = node.ownFirstVisibleId; + if (leftLastVisible !== undefined && + ownFirstVisible !== undefined && + leftLastVisible.peer === ownFirstVisible.peer && + leftLastVisible.counter + 1 === ownFirstVisible.counter) { + node.visibleIdRunCount -= 1; + } + const beforeRight = node.ownLastVisibleId ?? leftLastVisible; + const rightFirstVisible = right?.firstVisibleId; + if (beforeRight !== undefined && + rightFirstVisible !== undefined && + beforeRight.peer === rightFirstVisible.peer && + beforeRight.counter + 1 === rightFirstVisible.counter) { + node.visibleIdRunCount -= 1; + } +} +function recompute(node, _metrics) { + const left = node.left; + const right = node.right; + if (left === undefined && right === undefined) { + node.allCount = nodeLength(node); + node.visibleCount = node.ownVisibleCount; + node.visibleUtf16 = node.ownVisibleUtf16; + node.visibleUtf8 = node.ownVisibleUtf8; + node.firstVisibleId = node.ownFirstVisibleId; + node.lastVisibleId = node.ownLastVisibleId; + node.visibleIdRunCount = node.ownVisibleIdRunCount; + node.firstId = node.ownFirstId; + node.lastId = node.ownLastId; + node.idRunCount = node.ownIdRunCount; + node.allUtf16 = node.ownUtf16; + node.allUtf8 = node.ownUtf8; + return; + } + node.allCount = (left?.allCount ?? 0) + nodeLength(node) + (right?.allCount ?? 0); + node.allUtf16 = (left?.allUtf16 ?? 0) + node.ownUtf16 + (right?.allUtf16 ?? 0); + node.allUtf8 = (left?.allUtf8 ?? 0) + node.ownUtf8 + (right?.allUtf8 ?? 0); + recomputeVisibility(node); + node.firstId = left?.firstId ?? node.ownFirstId; + node.lastId = right?.lastId ?? node.ownLastId; + node.idRunCount = + (left?.idRunCount ?? 0) + node.ownIdRunCount + (right?.idRunCount ?? 0); + if (left !== undefined && + left.lastId.peer === node.ownFirstId.peer && + left.lastId.counter + 1 === node.ownFirstId.counter) { + node.idRunCount -= 1; + } + if (right !== undefined && + node.ownLastId.peer === right.firstId.peer && + node.ownLastId.counter + 1 === right.firstId.counter) { + node.idRunCount -= 1; + } + if (left !== undefined) + left.parent = node; + if (right !== undefined) + right.parent = node; +} +function applyNodeDeleted(node, metrics) { + if (node.lazyDeleted && node.visibleCount === 0) + return; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + setNodeDeleted(node, offset, true); + } + recomputeOwn(node, metrics); + node.visibleCount = 0; + node.visibleUtf16 = 0; + node.visibleUtf8 = 0; + node.firstVisibleId = undefined; + node.lastVisibleId = undefined; + node.visibleIdRunCount = 0; + node.lazyDeleted = true; + node.lazyVisible = false; +} +function applyNodeVisible(node, metrics) { + if (node.lazyVisible && node.visibleCount === node.allCount) + return; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + setNodeDeleted(node, offset, false); + } + recomputeOwn(node, metrics); + node.visibleCount = node.allCount; + node.visibleUtf16 = node.allUtf16; + node.visibleUtf8 = node.allUtf8; + node.firstVisibleId = node.firstId; + node.lastVisibleId = node.lastId; + node.visibleIdRunCount = node.idRunCount; + node.lazyDeleted = false; + node.lazyVisible = true; +} +function pushNodeDeletion(node, metrics) { + if (!node.lazyDeleted && !node.lazyVisible) + return; + if (node.left !== undefined) { + if (node.lazyDeleted) + applyNodeDeleted(node.left, metrics); + else + applyNodeVisible(node.left, metrics); + } + if (node.right !== undefined) { + if (node.lazyDeleted) + applyNodeDeleted(node.right, metrics); + else + applyNodeVisible(node.right, metrics); + } + node.lazyDeleted = false; + node.lazyVisible = false; +} +function materializeElementDeletion(element, metrics) { + const node = elementNode(element); + if (node === undefined) + return; + materializeNodeDeletion(node, metrics); +} +function materializeNodeDeletion(node, metrics) { + const path = []; + let current = node; + while (current !== undefined) { + path.push(current); + current = current.parent; + } + for (let index = path.length - 1; index >= 0; index -= 1) { + pushNodeDeletion(path[index], metrics); + } +} +function materializeAllDeletions(node, metrics) { + if (node === undefined) + return; + pushNodeDeletion(node, metrics); + materializeAllDeletions(node.left, metrics); + materializeAllDeletions(node.right, metrics); +} +function recomputeOwnIdRuns(node) { + node.ownFirstVisibleId = undefined; + node.ownLastVisibleId = undefined; + node.ownVisibleIdRunCount = 0; + let lastPeer; + let lastCounter = 0; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (nodeDeleted(node, offset)) + continue; + const peer = nodePeer(node, offset); + const counter = nodeCounter(node, offset); + if (lastPeer === undefined) { + node.ownFirstVisibleId = { peer, counter }; + node.ownVisibleIdRunCount = 1; + } + else if (!sequenceIdScalarsContinue(lastPeer, lastCounter, peer, counter)) { + node.ownVisibleIdRunCount += 1; + } + lastPeer = peer; + lastCounter = counter; + } + node.ownLastVisibleId = + lastPeer === undefined ? undefined : { peer: lastPeer, counter: lastCounter }; +} +function sequenceIdsContinue(left, right) { + return left.peer === right.peer && left.counter + 1 === right.counter; +} +function sequenceIdScalarsContinue(leftPeer, leftCounter, rightPeer, rightCounter) { + return leftPeer === rightPeer && leftCounter + 1 === rightCounter; +} +function normalizeSequenceIdRuns(runs) { + const sorted = [...runs].sort((left, right) => left.start.peer < right.start.peer + ? -1 + : left.start.peer > right.start.peer + ? 1 + : left.start.counter - right.start.counter); + const output = []; + for (const run of sorted) { + const previous = output.at(-1); + if (previous !== undefined && + previous.start.peer === run.start.peer && + run.start.counter <= previous.start.counter + previous.length) { + previous.length = + Math.max(previous.start.counter + previous.length, run.start.counter + run.length) - previous.start.counter; + } + else { + output.push({ start: { ...run.start }, length: run.length }); + } + } + return output; +} +function recomputeToRoot(node, metrics, includePhysical = true) { + let current = node; + while (current !== undefined) { + if (includePhysical) + recompute(current, metrics); + else + recomputeVisibility(current); + current = current.parent; + } +} +function recomputeAffected(node, affected, metrics) { + if (node === undefined || !affected.has(node)) + return; + recomputeAffected(node.left, affected, metrics); + recomputeAffected(node.right, affected, metrics); + recompute(node, metrics); +} +function merge(left, right, metrics) { + if (left === undefined) { + if (right !== undefined) + right.parent = undefined; + return right; + } + if (right === undefined) { + left.parent = undefined; + return left; + } + if (left.priority <= right.priority) { + pushNodeDeletion(left, metrics); + left.right = merge(left.right, right, metrics); + recompute(left, metrics); + left.parent = undefined; + return left; + } + pushNodeDeletion(right, metrics); + right.left = merge(left, right.left, metrics); + recompute(right, metrics); + right.parent = undefined; + return right; +} +function visitInOrder(root, visit) { + const stack = []; + let node = root; + while (node !== undefined || stack.length > 0) { + while (node !== undefined) { + stack.push(node); + node = node.left; + } + node = stack.pop(); + visit(node); + node = node.right; + } +} +function visitVisibleIdRuns(node, start, end, metrics, visit) { + if (node === undefined || start >= end) + return; + const total = visibleCount(node); + if (start === 0 && + end === total && + node.visibleIdRunCount === 1 && + node.firstVisibleId !== undefined) { + visit(node.firstVisibleId, total); + return; + } + pushNodeDeletion(node, metrics); + const leftCount = visibleCount(node.left); + if (start < leftCount) { + visitVisibleIdRuns(node.left, start, Math.min(end, leftCount), metrics, visit); + } + const ownCount = ownVisibleCount(node); + const ownStart = Math.max(0, start - leftCount); + const ownEnd = Math.min(ownCount, end - leftCount); + if (ownStart < ownEnd) { + if (ownStart === 0 && + ownEnd === ownCount && + node.ownVisibleIdRunCount === 1 && + node.ownFirstVisibleId !== undefined) { + visit(node.ownFirstVisibleId, ownCount); + } + else { + let visibleOffset = 0; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (nodeDeleted(node, offset)) + continue; + if (visibleOffset >= ownStart && visibleOffset < ownEnd) { + // The visit callback may retain the id, so each run start is a + // fresh object; only the element view allocation is avoided. + visit({ peer: nodePeer(node, offset), counter: nodeCounter(node, offset) }, 1); + } + visibleOffset += 1; + if (visibleOffset >= ownEnd) + break; + } + } + } + const rightStart = leftCount + ownCount; + if (end > rightStart) { + visitVisibleIdRuns(node.right, Math.max(0, start - rightStart), end - rightStart, metrics, visit); + } +} +function counterRangesByPeer(runs) { + const byPeer = new Map(); + for (const run of runs) { + if (run.length <= 0) + continue; + let ranges = byPeer.get(run.start.peer); + if (ranges === undefined) { + ranges = []; + byPeer.set(run.start.peer, ranges); + } + ranges.push({ start: run.start.counter, end: run.start.counter + run.length }); + } + for (const ranges of byPeer.values()) { + ranges.sort((left, right) => left.start - right.start); + let write = 0; + for (const range of ranges) { + const previous = ranges[write - 1]; + if (previous !== undefined && range.start <= previous.end) { + previous.end = Math.max(previous.end, range.end); + } + else { + ranges[write++] = range; + } + } + ranges.length = write; + } + return byPeer; +} +function counterRangeRelation(targets, peer, start, end) { + const ranges = targets.get(peer); + if (ranges === undefined) + return 0; + let low = 0; + let high = ranges.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (ranges[middle].end <= start) + low = middle + 1; + else + high = middle; + } + const range = ranges[low]; + if (range === undefined || range.start >= end) + return 0; + return range.start <= start && range.end >= end ? 1 : -1; +} +function mapCounterRangeRelation(targets) { + return (peer, start, end) => counterRangeRelation(targets, peer, start, end); +} +/** + * Scalar relation for the common single-run delete: avoids building a + * Map and the per-node bigint-keyed lookup. + */ +function singleCounterRangeRelation(targetPeer, targetStart, targetEnd) { + return (peer, start, end) => { + if (peer !== targetPeer || targetStart >= end || targetEnd <= start) + return 0; + return targetStart <= start && targetEnd >= end ? 1 : -1; + }; +} +function visitVisibleMetricRangesForIds(node, base, metric, targets, metrics, visit) { + if (node === undefined || node.visibleCount === 0) + return; + if (node.visibleIdRunCount === 1 && node.firstVisibleId !== undefined) { + const relation = counterRangeRelation(targets, node.firstVisibleId.peer, node.firstVisibleId.counter, node.firstVisibleId.counter + node.visibleCount); + if (relation === 0) + return; + if (relation === 1) { + visit(base, base + visibleMetric(node, metric)); + return; + } + } + pushNodeDeletion(node, metrics); + visitVisibleMetricRangesForIds(node.left, base, metric, targets, metrics, visit); + let ownBase = base + visibleMetric(node.left, metric); + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (nodeDeleted(node, offset)) + continue; + const length = nodeMetrics(node, offset, metrics)[metric]; + const counter = nodeCounter(node, offset); + if (counterRangeRelation(targets, nodePeer(node, offset), counter, counter + 1) === 1) { + visit(ownBase, ownBase + length); + } + ownBase += length; + } + visitVisibleMetricRangesForIds(node.right, ownBase, metric, targets, metrics, visit); +} +function countPhysicalIdsInRanges(node, targets) { + if (node === undefined) + return 0; + if (node.idRunCount === 1) { + const relation = counterRangeRelation(targets, node.firstId.peer, node.firstId.counter, node.firstId.counter + node.allCount); + if (relation === 0) + return 0; + if (relation === 1) + return node.allCount; + } + let count = countPhysicalIdsInRanges(node.left, targets); + for (let offset = 0; offset < nodeLength(node); offset += 1) { + const counter = nodeCounter(node, offset); + if (counterRangeRelation(targets, nodePeer(node, offset), counter, counter + 1) === 1) { + count += 1; + } + } + return count + countPhysicalIdsInRanges(node.right, targets); +} +function markIdRunsDeleted(node, relate, metrics) { + if (node === undefined) + return { found: false, changed: false }; + if (node.idRunCount === 1) { + const relation = relate(node.firstId.peer, node.firstId.counter, node.firstId.counter + node.allCount); + if (relation === 0) + return { found: false, changed: false }; + if (relation === 1) { + const changed = node.visibleCount > 0; + applyNodeDeleted(node, metrics); + return { found: true, changed }; + } + } + pushNodeDeletion(node, metrics); + const left = markIdRunsDeleted(node.left, relate, metrics); + let ownFound = false; + let ownChanged = false; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + const counter = nodeCounter(node, offset); + if (relate(nodePeer(node, offset), counter, counter + 1) !== 1) { + continue; + } + ownFound = true; + if (!nodeDeleted(node, offset)) { + setNodeDeleted(node, offset, true); + ownChanged = true; + } + } + if (ownChanged) + recomputeOwn(node, metrics); + const right = markIdRunsDeleted(node.right, relate, metrics); + const changed = left.changed || ownChanged || right.changed; + if (changed) + recomputeVisibility(node); + return { + found: left.found || ownFound || right.found, + changed, + }; +} +function markIdRunsVisible(node, relate, metrics) { + if (node === undefined) + return { found: false, changed: false }; + if (node.idRunCount === 1) { + const relation = relate(node.firstId.peer, node.firstId.counter, node.firstId.counter + node.allCount); + if (relation === 0) + return { found: false, changed: false }; + if (relation === 1) { + const changed = node.visibleCount < node.allCount; + applyNodeVisible(node, metrics); + return { found: true, changed }; + } + } + pushNodeDeletion(node, metrics); + const left = markIdRunsVisible(node.left, relate, metrics); + let ownFound = false; + let ownChanged = false; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + const counter = nodeCounter(node, offset); + if (relate(nodePeer(node, offset), counter, counter + 1) !== 1) { + continue; + } + ownFound = true; + if (nodeDeleted(node, offset)) { + setNodeDeleted(node, offset, false); + ownChanged = true; + } + } + if (ownChanged) + recomputeOwn(node, metrics); + const right = markIdRunsVisible(node.right, relate, metrics); + const changed = left.changed || ownChanged || right.changed; + if (changed) + recomputeVisibility(node); + return { + found: left.found || ownFound || right.found, + changed, + }; +} +function visitVisibleRange(node, start, end, metrics, visit) { + if (node === undefined || start >= end) + return true; + pushNodeDeletion(node, metrics); + const leftCount = visibleCount(node.left); + if (start < leftCount) { + if (!visitVisibleRange(node.left, start, Math.min(end, leftCount), metrics, visit)) { + return false; + } + } + let ownOffset = leftCount; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (nodeDeleted(node, offset)) + continue; + if (start <= ownOffset && + end > ownOffset && + visit(nodeElement(node, offset)) === false) { + return false; + } + ownOffset += 1; + if (ownOffset >= end) + break; + } + if (end > ownOffset) { + return visitVisibleRange(node.right, Math.max(0, start - ownOffset), end - ownOffset, metrics, visit); + } + return true; +} +function collectCausalRange(node, start, end, output, count, isVisible, metrics) { + if (node === undefined || start >= end) + return; + pushNodeDeletion(node, metrics); + const leftCount = count(node.left); + if (start < leftCount) { + collectCausalRange(node.left, start, Math.min(end, leftCount), output, count, isVisible, metrics); + } + let ownOffset = leftCount; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (!isVisible(node, offset)) + continue; + if (start <= ownOffset && end > ownOffset) + output.push(nodeElement(node, offset)); + ownOffset += 1; + if (ownOffset >= end) + break; + } + if (end > ownOffset) { + collectCausalRange(node.right, Math.max(0, start - ownOffset), end - ownOffset, output, count, isVisible, metrics); + } +} +function collectCausalIdRuns(node, start, end, output, count, isVisible) { + if (node === undefined || start >= end) + return; + const nodeCount = count(node); + if (start === 0 && + end === nodeCount && + nodeCount === node.allCount && + node.idRunCount === 1) { + appendSequenceIdRun(output, node.firstId, node.allCount); + return; + } + const leftCount = count(node.left); + if (start < leftCount) { + collectCausalIdRuns(node.left, start, Math.min(end, leftCount), output, count, isVisible); + } + let ownOffset = leftCount; + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (!isVisible(node, offset)) + continue; + if (start <= ownOffset && end > ownOffset) { + // appendSequenceIdRun copies the start id before retaining it. + scratchSequenceRunId.peer = nodePeer(node, offset); + scratchSequenceRunId.counter = nodeCounter(node, offset); + appendSequenceIdRun(output, scratchSequenceRunId, 1); + } + ownOffset += 1; + if (ownOffset >= end) + break; + } + if (end > ownOffset) { + collectCausalIdRuns(node.right, Math.max(0, start - ownOffset), end - ownOffset, output, count, isVisible); + } +} +function appendSequenceIdRun(output, start, length) { + if (length <= 0) + return; + const previous = output.at(-1); + if (previous !== undefined && + previous.start.peer === start.peer && + previous.start.counter + previous.length === start.counter) { + output[output.length - 1] = { + start: previous.start, + length: previous.length + length, + }; + } + else { + output.push({ start: { ...start }, length }); + } +} +function firstPhysicalElement(node) { + if (node === undefined) + return undefined; + while (node.left !== undefined) + node = node.left; + return nodeElement(node, 0); +} +function writeFirstPhysicalId(node, output) { + if (node === undefined) { + output.rightPeer = undefined; + output.rightCounter = 0; + return; + } + while (node.left !== undefined) + node = node.left; + output.rightPeer = nodePeer(node, 0); + output.rightCounter = nodeCounter(node, 0); +} +function nextPhysicalElement(node, offset) { + if (offset + 1 < nodeLength(node)) + return nodeElement(node, offset + 1); + const fromRight = firstPhysicalElement(node.right); + if (fromRight !== undefined) + return fromRight; + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.left) + return nodeElement(current.parent, 0); + current = current.parent; + } + return undefined; +} +function writeNextPhysicalId(node, offset, output) { + if (offset + 1 < nodeLength(node)) { + output.rightPeer = nodePeer(node, offset + 1); + output.rightCounter = nodeCounter(node, offset + 1); + return; + } + if (node.right !== undefined) { + writeFirstPhysicalId(node.right, output); + return; + } + let current = node; + while (current.parent !== undefined) { + if (current === current.parent.left) { + output.rightPeer = nodePeer(current.parent, 0); + output.rightCounter = nodeCounter(current.parent, 0); + return; + } + current = current.parent; + } + output.rightPeer = undefined; + output.rightCounter = 0; +} +function firstVisibleElement(node, metrics) { + while (node !== undefined && node.visibleCount > 0) { + pushNodeDeletion(node, metrics); + if (visibleCount(node.left) > 0) { + node = node.left; + continue; + } + for (let offset = 0; offset < nodeLength(node); offset += 1) { + if (!nodeDeleted(node, offset)) + return nodeElement(node, offset); + } + node = node.right; + } + return undefined; +} +function lastVisibleElement(node, metrics) { + while (node !== undefined && node.visibleCount > 0) { + pushNodeDeletion(node, metrics); + if (visibleCount(node.right) > 0) { + node = node.right; + continue; + } + for (let offset = nodeLength(node) - 1; offset >= 0; offset -= 1) { + if (!nodeDeleted(node, offset)) + return nodeElement(node, offset); + } + node = node.left; + } + return undefined; +} +function rightmost(node) { + while (node.right !== undefined) + node = node.right; + return node; +} +function versionsEqual(left, right) { + if (left === undefined || left.size !== right.size) + return false; + for (const [peer, counter] of left) { + if (right.get(peer) !== counter) + return false; + } + return true; +} +const MAX_COUNTER_DENSE_GAP = 4096; +const COUNTER_PAGE_SIZE = 1024; +class PagedCounterStore { + #pages = new Map(); + #pageIndexes = []; + set(counter, value) { + // counter is non-negative; COUNTER_PAGE_SIZE is 1,024. + const pageIndex = (counter / COUNTER_PAGE_SIZE) | 0; + let page = this.#pages.get(pageIndex); + if (page === undefined) { + page = []; + this.#pages.set(pageIndex, page); + const index = lowerBoundNumber(this.#pageIndexes, pageIndex); + this.#pageIndexes.splice(index, 0, pageIndex); + } + page[counter - pageIndex * COUNTER_PAGE_SIZE] = value; + } + some(start, end, predicate) { + if (start >= end) + return false; + const firstPage = (start / COUNTER_PAGE_SIZE) | 0; + const lastPage = ((end - 1) / COUNTER_PAGE_SIZE) | 0; + for (let index = lowerBoundNumber(this.#pageIndexes, firstPage); index < this.#pageIndexes.length && this.#pageIndexes[index] <= lastPage; index += 1) { + const pageIndex = this.#pageIndexes[index]; + const page = this.#pages.get(pageIndex); + const pageStart = pageIndex * COUNTER_PAGE_SIZE; + const offsetStart = Math.max(0, start - pageStart); + const offsetEnd = Math.min(page.length, end - pageStart); + for (let offset = offsetStart; offset < offsetEnd; offset += 1) { + const value = page[offset]; + if (value !== undefined && predicate(value)) + return true; + } + } + return false; + } +} +class CounterStore { + #dense = []; + #sparse = new Map(); + #sparseCounters = new CounterIndex(); + get(counter) { + return counter < this.#dense.length + ? this.#dense[counter] + : this.#sparse.get(counter); + } + has(counter) { + return this.get(counter) !== undefined; + } + set(counter, value) { + if (counter < 4294967295 && counter <= this.#dense.length + MAX_COUNTER_DENSE_GAP) { + this.#dense[counter] = value; + return; + } + if (!this.#sparse.has(counter)) + this.#sparseCounters.add(counter); + this.#sparse.set(counter, value); + } + reserveDense(endExclusive) { + if (this.#sparse.size > 0) { + throw new Error("cannot reserve dense sequence counters after sparse insertion"); + } + if (endExclusive > this.#dense.length) + this.#dense.length = endExclusive; + } + forEach(start, end, visit) { + if (start >= end) + return; + const denseEnd = Math.min(end, this.#dense.length); + for (let counter = Math.max(0, start); counter < denseEnd; counter += 1) { + const value = this.#dense[counter]; + if (value !== undefined) + visit(value, counter); + } + this.#sparseCounters.forEach(start, end, (counter) => { + visit(this.#sparse.get(counter), counter); + }); + } +} +class CounterIndex { + #ranges = []; + #singletons = []; + #outOfOrder = new CounterRangeTree(); + #lastCounter; + #tailIsRange = false; + add(counter) { + const last = this.#lastCounter; + if (last === undefined) { + this.#singletons.push(counter); + this.#lastCounter = counter; + return; + } + if (counter > last) { + if (counter === last + 1) { + if (this.#tailIsRange) { + this.#ranges[this.#ranges.length - 1] = counter + 1; + } + else { + this.#ranges.push(this.#singletons.pop(), counter + 1); + this.#tailIsRange = true; + } + } + else { + this.#singletons.push(counter); + this.#tailIsRange = false; + } + this.#lastCounter = counter; + return; + } + if (counter === last || containsCounter(this.#ranges, this.#singletons, counter)) { + return; + } + this.#outOfOrder.add(counter); + } + forEach(start, end, visit) { + if (start >= end) + return; + const index = lowerBoundCounterRange(this.#ranges, start); + if (index > 0) { + const previousEnd = this.#ranges[index * 2 - 1]; + if (previousEnd > start) { + visitCounterInterval(this.#ranges[(index - 1) * 2], previousEnd, start, end, visit); + } + } + for (let rangeIndex = index; rangeIndex < this.#ranges.length / 2 && this.#ranges[rangeIndex * 2] < end; rangeIndex += 1) { + visitCounterInterval(this.#ranges[rangeIndex * 2], this.#ranges[rangeIndex * 2 + 1], start, end, visit); + } + for (let singletonIndex = lowerBoundNumber(this.#singletons, start); singletonIndex < this.#singletons.length && this.#singletons[singletonIndex] < end; singletonIndex += 1) { + visit(this.#singletons[singletonIndex]); + } + this.#outOfOrder.forEach(start, end, visit); + } +} +class CounterRangeTree { + #ranges = new OrderedIndex((left, right) => left.start - right.start); + add(counter) { + const range = { start: counter, end: counter + 1 }; + const index = this.#ranges.lowerBound(range); + const previous = this.#ranges.at(index - 1); + if (previous !== undefined && counter < previous.end) + return; + const next = this.#ranges.at(index); + if (next?.start === counter) + return; + const extendsPrevious = previous?.end === counter; + const extendsNext = next?.start === counter + 1; + if (extendsPrevious && extendsNext) { + previous.end = next.end; + this.#ranges.delete(next); + } + else if (extendsPrevious) { + previous.end += 1; + } + else if (extendsNext) { + this.#ranges.delete(next); + next.start = counter; + this.#ranges.add(next); + } + else { + this.#ranges.add(range); + } + } + forEach(start, end, visit) { + if (start >= end) + return; + const lower = { start, end: start + 1 }; + const index = this.#ranges.lowerBound(lower); + const previous = this.#ranges.at(index - 1); + if (previous !== undefined && previous.end > start) { + visitCounterRange(previous, start, end, visit); + } + this.#ranges.forEachFrom(lower, (range) => { + if (range.start >= end) + return false; + visitCounterRange(range, start, end, visit); + }); + } +} +function lowerBoundCounterRange(ranges, counter) { + let low = 0; + let high = ranges.length / 2; + while (low < high) { + const middle = (low + high) >>> 1; + if (ranges[middle * 2] < counter) + low = middle + 1; + else + high = middle; + } + return low; +} +function lowerBoundNumber(values, value) { + let low = 0; + let high = values.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (values[middle] < value) + low = middle + 1; + else + high = middle; + } + return low; +} +function containsCounter(ranges, singletons, counter) { + const singletonIndex = lowerBoundNumber(singletons, counter); + if (singletonIndex < singletons.length && singletons[singletonIndex] === counter) { + return true; + } + const rangeIndex = lowerBoundCounterRange(ranges, counter); + if (rangeIndex * 2 < ranges.length && ranges[rangeIndex * 2] === counter) + return true; + return rangeIndex > 0 && counter < ranges[rangeIndex * 2 - 1]; +} +function visitCounterInterval(rangeStart, rangeEnd, start, end, visit) { + const boundedEnd = Math.min(rangeEnd, end); + for (let counter = Math.max(rangeStart, start); counter < boundedEnd; counter += 1) { + visit(counter); + } +} +function visitCounterRange(range, start, end, visit) { + const rangeEnd = Math.min(range.end, end); + for (let counter = Math.max(range.start, start); counter < rangeEnd; counter += 1) { + visit(counter); + } +} diff --git a/loro-js/src/runtime/text-style-index.js b/loro-js/src/runtime/text-style-index.js new file mode 100644 index 000000000..731d597a1 --- /dev/null +++ b/loro-js/src/runtime/text-style-index.js @@ -0,0 +1,336 @@ +import { OrderedIndex } from "./ordered-index"; +export class TextStyleIndex { + #segmentsByPeer = new Map(); + #emptyMetas = new Map(); + get isEmpty() { + return this.#segmentsByPeer.size === 0; + } + add(runs, key, meta) { + for (const run of normalizeRuns(runs)) { + const end = run.start.counter + run.length; + if (run.length <= 0) + continue; + const segments = this.#segments(run.start.peer); + this.#ensureBoundary(segments, run.start.counter); + this.#ensureBoundary(segments, end); + const existing = []; + const first = segments._lowerBoundBy((segment) => segment.start - run.start.counter); + segments._forEachFromIndex(first, (segment) => { + if (segment.start >= end) + return false; + existing.push(segment); + }); + let cursor = run.start.counter; + for (const segment of existing) { + if (cursor < segment.start) { + const gap = { + start: cursor, + end: segment.start, + histories: new Map(), + latestMetas: undefined, + }; + this.#insertMeta(gap, key, meta); + segments.add(gap); + } + this.#insertMeta(segment, key, meta); + cursor = segment.end; + } + if (cursor < end) { + const gap = { + start: cursor, + end, + histories: new Map(), + latestMetas: undefined, + }; + this.#insertMeta(gap, key, meta); + segments.add(gap); + } + } + } + historyAt(id, key) { + return this.#segmentAt(id)?.histories.get(key); + } + metasAt(id, version) { + const segment = this.#segmentAt(id); + if (segment === undefined) + return this.#emptyMetas; + if (version === undefined) + return this.#latestMetas(segment); + const metas = new Map(); + for (const [key, history] of segment.histories) { + const meta = latestIncluded(history, version); + if (meta !== undefined) + metas.set(key, meta); + } + return metas; + } + resolver(version) { + const lastByPeer = new Map(); + const resolved = new Map(); + return (id) => { + let lookup = lastByPeer.get(id.peer); + if (lookup === undefined || id.counter < lookup.start || id.counter >= lookup.end) { + lookup = this.#lookupAt(id); + lastByPeer.set(id.peer, lookup); + } + const segment = lookup.segment; + if (segment === undefined) + return this.#emptyMetas; + if (version === undefined) + return this.#latestMetas(segment); + const cached = resolved.get(segment); + if (cached !== undefined) + return cached; + const metas = new Map(); + for (const [key, history] of segment.histories) { + const meta = latestIncluded(history, version); + if (meta !== undefined) + metas.set(key, meta); + } + resolved.set(segment, metas); + return metas; + }; + } + rangeHasKey(runs, key, version) { + for (const run of normalizeRuns(runs)) { + const segments = this.#segmentsByPeer.get(run.start.peer); + if (segments === undefined) + continue; + const end = run.start.counter + run.length; + const first = Math.max(0, segments._lowerBoundBy((segment) => segment.start - run.start.counter) - 1); + let found = false; + segments._forEachFromIndex(first, (segment) => { + if (segment.start >= end) + return false; + if (segment.end <= run.start.counter) + return; + const meta = latestIncluded(segment.histories.get(key), version); + if (meta !== undefined && meta.value !== null) { + found = true; + return false; + } + }); + if (found) + return true; + } + return false; + } + transitions(runs, key, beforeVersion, afterVersion) { + const transitions = []; + const append = (start, length, before, after) => { + if (before === after || length === 0) + return; + const previous = transitions.at(-1); + if (previous !== undefined && + previous.before === before && + previous.after === after && + previous.run.start.peer === start.peer && + previous.run.start.counter + previous.run.length === start.counter) { + transitions[transitions.length - 1] = { + run: { start: previous.run.start, length: previous.run.length + length }, + before, + after, + }; + } + else { + transitions.push({ run: { start: { ...start }, length }, before, after }); + } + }; + for (const run of normalizeRuns(runs)) { + const segments = this.#segmentsByPeer.get(run.start.peer); + if (segments === undefined) + continue; + const end = run.start.counter + run.length; + const first = Math.max(0, segments._lowerBoundBy((segment) => segment.start - run.start.counter) - 1); + segments._forEachFromIndex(first, (segment) => { + if (segment.start >= end) + return false; + const start = Math.max(run.start.counter, segment.start); + const segmentEnd = Math.min(end, segment.end); + if (start >= segmentEnd) + return; + append({ peer: run.start.peer, counter: start }, segmentEnd - start, latestIncluded(segment.histories.get(key), beforeVersion), latestIncluded(segment.histories.get(key), afterVersion)); + }); + } + return transitions; + } + runsContainMeta(runs, key, id) { + for (const run of runs) { + const segments = this.#segmentsByPeer.get(run.start.peer); + if (segments === undefined) + return false; + const end = run.start.counter + run.length; + let cursor = run.start.counter; + const first = Math.max(0, segments._lowerBoundBy((segment) => segment.start - cursor) - 1); + let covered = true; + segments._forEachFromIndex(first, (segment) => { + if (cursor >= end) + return false; + if (segment.start > cursor || segment.start >= end) { + covered = false; + return false; + } + if (segment.end <= cursor) + return; + if (!segment.histories + .get(key) + ?.some((item) => item.startId.peer === id.peer && item.startId.counter === id.counter)) { + covered = false; + return false; + } + cursor = Math.min(end, segment.end); + }); + if (!covered || cursor < end) + return false; + } + return true; + } + reset() { + this.#segmentsByPeer.clear(); + } + #segments(peer) { + let segments = this.#segmentsByPeer.get(peer); + if (segments === undefined) { + segments = new OrderedIndex((left, right) => left.start - right.start); + this.#segmentsByPeer.set(peer, segments); + } + return segments; + } + #segmentAt(id) { + return this.#lookupAt(id).segment; + } + #lookupAt(id) { + const segments = this.#segmentsByPeer.get(id.peer); + if (segments === undefined) { + return { + start: Number.MIN_SAFE_INTEGER, + end: Number.MAX_SAFE_INTEGER, + segment: undefined, + }; + } + const nextIndex = segments._lowerBoundBy((segment) => segment.start - (id.counter + 1)); + const previous = segments.at(nextIndex - 1); + if (previous !== undefined && id.counter < previous.end) { + return { start: previous.start, end: previous.end, segment: previous }; + } + return { + start: previous?.end ?? Number.MIN_SAFE_INTEGER, + end: segments.at(nextIndex)?.start ?? Number.MAX_SAFE_INTEGER, + segment: undefined, + }; + } + #ensureBoundary(segments, counter) { + const index = segments._lowerBoundBy((segment) => segment.start - counter); + if (segments.at(index)?.start === counter) + return; + const previous = segments.at(index - 1); + if (previous === undefined || counter >= previous.end) + return; + segments.delete(previous); + const right = { + start: counter, + end: previous.end, + histories: cloneHistories(previous.histories), + latestMetas: previous.latestMetas, + }; + previous.end = counter; + segments.add(previous); + segments.add(right); + } + #insertMeta(segment, key, meta) { + let history = segment.histories.get(key); + if (history === undefined) { + history = []; + segment.histories.set(key, history); + } + const index = lowerBoundMeta(history, meta); + if (sameMeta(history[index], meta)) + return; + history.splice(index, 0, meta); + segment.latestMetas = undefined; + } + #latestMetas(segment) { + if (segment.latestMetas !== undefined) + return segment.latestMetas; + const metas = new Map(); + for (const [key, history] of segment.histories) { + const meta = history.at(-1); + if (meta !== undefined) + metas.set(key, meta); + } + segment.latestMetas = metas; + return metas; + } +} +function cloneHistories(histories) { + return new Map([...histories].map(([key, history]) => [key, [...history]])); +} +function normalizeRuns(runs) { + // Single-run input is the common case and is already normalized; returning + // it as-is skips the filter/map/sort chain. Callers only read the result. + if (runs.length === 0) + return []; + if (runs.length === 1) { + const run = runs[0]; + return run.length > 0 ? [run] : []; + } + const sorted = runs + .filter((run) => run.length > 0) + .map((run) => ({ start: { ...run.start }, length: run.length })) + .sort((left, right) => left.start.peer < right.start.peer + ? -1 + : left.start.peer > right.start.peer + ? 1 + : left.start.counter - right.start.counter); + const merged = []; + for (const run of sorted) { + const previous = merged.at(-1); + const end = run.start.counter + run.length; + if (previous !== undefined && + previous.start.peer === run.start.peer && + run.start.counter <= previous.start.counter + previous.length) { + previous.length = + Math.max(previous.start.counter + previous.length, end) - previous.start.counter; + } + else { + merged.push(run); + } + } + return merged; +} +function compareMeta(left, right) { + return (left.lamport - right.lamport || + (left.startId.peer < right.startId.peer + ? -1 + : left.startId.peer > right.startId.peer + ? 1 + : 0)); +} +function lowerBoundMeta(history, meta) { + let low = 0; + let high = history.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compareMeta(history[middle], meta) < 0) + low = middle + 1; + else + high = middle; + } + return low; +} +function latestIncluded(history, version) { + if (history === undefined) + return undefined; + if (version === undefined) + return history.at(-1); + for (let index = history.length - 1; index >= 0; index -= 1) { + const meta = history[index]; + if (meta.startId.counter < (version.get(meta.startId.peer) ?? 0)) + return meta; + } + return undefined; +} +function sameMeta(left, right) { + return (left?.startId.peer === right.startId.peer && + left.startId.counter === right.startId.counter); +} diff --git a/loro-js/src/runtime/types.js b/loro-js/src/runtime/types.js new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/loro-js/src/runtime/types.js @@ -0,0 +1 @@ +export {}; diff --git a/loro-js/src/runtime/undo.js b/loro-js/src/runtime/undo.js new file mode 100644 index 000000000..d69448098 --- /dev/null +++ b/loro-js/src/runtime/undo.js @@ -0,0 +1,274 @@ +class UndoDeque { + // Array + head offset: capacity is capped by maxUndoSteps, so a Map of + // slot indexes only costs extra entries per push/pop. + #items = []; + #start = 0; + get length() { + return this.#items.length - this.#start; + } + push(item) { + this.#items.push(item); + } + pop() { + if (this.length === 0) + return undefined; + const item = this.#items.pop(); + if (this.length === 0) + this.clear(); + return item; + } + peek() { + return this.length === 0 ? undefined : this.#items[this.#items.length - 1]; + } + trimFront(length) { + while (this.length > length) + this.#start += 1; + if (this.#start === 0) + return; + if (this.#start >= this.#items.length) { + this.clear(); + } + else if (this.#start >= 32 && this.#start * 2 >= this.#items.length) { + // Release the trimmed slots once they dominate the backing array. + this.#items.copyWithin(0, this.#start); + this.#items.length -= this.#start; + this.#start = 0; + } + } + clear() { + this.#items.length = 0; + this.#start = 0; + } +} +const EMPTY_META = { value: null, cursors: [] }; +export class UndoManager { + #doc; + #peer; + #undo = new UndoDeque(); + #redo = []; + #excludeOriginPrefixes = new Set(); + #remoteTargets = new Set(); + #mergeInterval; + #maxUndoSteps; + #onPush; + #onPop; + #applying = false; + #groupDepth = 0; + #unsubscribe; + constructor(doc, config = {}) { + this.#doc = doc; + this.#peer = doc.peerIdStr; + this.#mergeInterval = config.mergeInterval ?? 1000; + this.#maxUndoSteps = config.maxUndoSteps ?? 100; + this.#onPush = config.onPush; + this.#onPop = config.onPop; + for (const prefix of config.excludeOriginPrefixes ?? []) { + this.#excludeOriginPrefixes.add(prefix); + } + this.#unsubscribe = doc.subscribe((event) => this.#record(event)); + } + free() { + this.destroy(); + } + undo() { + const item = this.#undo.pop(); + if (item === undefined) + return false; + try { + const redo = this.#invert(item, true); + if (redo !== undefined) + this.#redo.push(redo); + } + catch (error) { + this.#undo.push(item); + throw error; + } + return true; + } + redo() { + const item = this.#redo.pop(); + if (item === undefined) + return false; + try { + const undo = this.#invert(item, false); + if (undo !== undefined) + this.#pushUndo(undo, false); + } + catch (error) { + this.#redo.push(item); + throw error; + } + return true; + } + peer() { + return this.#peer; + } + groupStart() { + this.#groupDepth += 1; + } + groupEnd() { + if (this.#groupDepth > 0) + this.#groupDepth -= 1; + } + canUndo() { + return this.#undo.length > 0; + } + canRedo() { + return this.#redo.length > 0; + } + topUndoValue() { + return this.#undo.peek()?.meta.value; + } + topRedoValue() { + return this.#redo.at(-1)?.meta.value; + } + setMaxUndoSteps(steps) { + if (!Number.isSafeInteger(steps) || steps < 0) { + throw new RangeError("max undo steps must be a nonnegative integer"); + } + this.#maxUndoSteps = steps; + this.#trimUndo(); + } + setMergeInterval(interval) { + if (!Number.isFinite(interval) || interval < 0) { + throw new RangeError("merge interval must be nonnegative"); + } + this.#mergeInterval = interval; + } + addExcludeOriginPrefix(prefix) { + this.#excludeOriginPrefixes.add(prefix); + } + setOnPush(callback) { + this.#onPush = callback; + } + setOnPop(callback) { + this.#onPop = callback; + } + clear() { + this.#undo.clear(); + this.#redo.length = 0; + this.#remoteTargets.clear(); + } + clearUndo() { + this.#undo.clear(); + } + clearRedo() { + this.#redo.length = 0; + } + destroy() { + this.#unsubscribe(); + this.clear(); + } + #record(event) { + if (this.#applying) + return; + const targets = new Set(); + for (const { target } of event.events) + targets.add(target); + if (event.by === "checkout") { + this.clear(); + return; + } + if (event.by === "import") { + for (const target of targets) + this.#remoteTargets.add(target); + return; + } + if (event.origin !== undefined && this.#excludedOrigin(event.origin)) { + for (const target of targets) + this.#remoteTargets.add(target); + return; + } + const spans = this.#doc.findIdSpansBetween(event.from, event.to).forward; + const span = spans.find(({ peer }) => peer === this.#doc.peerIdStr) ?? + (spans.length === 1 ? spans[0] : undefined); + if (span === undefined || span.length === 0) + return; + this.#peer = span.peer; + const now = Date.now(); + const item = { + peer: span.peer, + range: { start: span.counter, end: span.counter + span.length }, + meta: this.#onPush?.(true, { start: span.counter, end: span.counter + span.length }, event) ?? EMPTY_META, + timestamp: now, + targets, + }; + const previous = this.#undo.peek(); + let conflictsWithRemote = false; + if (previous !== undefined) { + for (const target of previous.targets) { + if (this.#remoteTargets.has(target)) { + conflictsWithRemote = true; + break; + } + } + } + const merge = previous !== undefined && + previous.peer === item.peer && + previous.range.end === item.range.start && + !conflictsWithRemote && + (this.#groupDepth > 0 || + (this.#mergeInterval > 0 && now - previous.timestamp < this.#mergeInterval)); + if (merge) { + previous.range = { start: previous.range.start, end: item.range.end }; + previous.meta = item.meta; + previous.timestamp = now; + for (const target of item.targets) + previous.targets.add(target); + } + else { + this.#pushUndo(item, false); + } + this.#redo.length = 0; + this.#remoteTargets.clear(); + } + #invert(item, isUndo) { + const before = this.#doc.frontiers(); + this.#applying = true; + try { + this.#doc._undoIdSpan(item.peer, item.range); + this.#doc.commit({ origin: isUndo ? "undo" : "redo" }); + const after = this.#doc.frontiers(); + const spans = this.#doc.findIdSpansBetween(before, after).forward; + const span = spans.find(({ peer }) => peer === this.#doc.peerIdStr) ?? + (spans.length === 1 ? spans[0] : undefined); + const poppedMeta = { + value: item.meta.value, + cursors: this.#doc._transformUndoCursors(item.meta.cursors), + }; + this.#onPop?.(isUndo, poppedMeta, item.range); + if (span === undefined || span.length === 0) + return undefined; + const range = { start: span.counter, end: span.counter + span.length }; + this.#peer = span.peer; + return { + peer: span.peer, + range, + meta: this.#onPush?.(!isUndo, range) ?? EMPTY_META, + timestamp: Date.now(), + targets: new Set(item.targets), + }; + } + finally { + this.#applying = false; + } + } + #pushUndo(item, clearRedo) { + this.#undo.push(item); + this.#trimUndo(); + if (clearRedo) + this.#redo.length = 0; + } + #excludedOrigin(origin) { + if (this.#excludeOriginPrefixes.size === 0) + return false; + for (const prefix of this.#excludeOriginPrefixes) { + if (origin.startsWith(prefix)) + return true; + } + return false; + } + #trimUndo() { + this.#undo.trimFront(this.#maxUndoSteps); + } +} diff --git a/loro-js/src/runtime/version-vector.js b/loro-js/src/runtime/version-vector.js new file mode 100644 index 000000000..9acbbbb02 --- /dev/null +++ b/loro-js/src/runtime/version-vector.js @@ -0,0 +1,112 @@ +import { decodePostcardVersionVector, encodePostcardVersionVector, } from "../codec/version"; +import { formatOpId, parseOpId, parsePeerId, peerIdToString } from "./ids"; +export class VersionVector { + #values = new Map(); + constructor(input) { + if (input == null) + return; + if (input instanceof Uint8Array) { + for (const id of decodePostcardVersionVector(input)) + this.#values.set(id.peer, id.counter); + return; + } + if (input instanceof VersionVector) { + for (const [peer, counter] of input.#values) + this.#values.set(peer, counter); + return; + } + const entries = input instanceof Map ? input.entries() : Object.entries(input); + for (const [peer, counter] of entries) + this.set(peer, counter); + } + free() { } + static parseJSON(input) { + return new VersionVector(input); + } + static decode(bytes) { + return new VersionVector(bytes); + } + encode() { + return encodePostcardVersionVector(this.codecEntries()); + } + toJSON() { + return new Map([...this.#values].map(([peer, counter]) => [peerIdToString(peer), counter])); + } + get(peer) { + return this.#values.get(parsePeerId(peer)); + } + compare(other) { + let less = false; + let greater = false; + for (const [peer, left] of this.#values) { + const right = other.#values.get(peer) ?? 0; + if (left < right) + less = true; + else if (left > right) + greater = true; + if (less && greater) + return undefined; + } + // Peers present only in the other vector compare as 0 on this side. + for (const [peer, right] of other.#values) { + if (this.#values.has(peer)) + continue; + if (right > 0) + less = true; + else if (right < 0) + greater = true; + if (less && greater) + return undefined; + } + return less ? -1 : greater ? 1 : 0; + } + setEnd(id) { + const parsed = parseOpId(id); + this.set(parsed.peer, parsed.counter); + } + setLast(id) { + const parsed = parseOpId(id); + this.set(parsed.peer, parsed.counter + 1); + } + remove(peer) { + this.#values.delete(parsePeerId(peer)); + } + length() { + return this.#values.size; + } + clone() { + return new VersionVector(this); + } + codecEntries() { + const entries = this._codecEntriesUnsorted(); + entries.sort((left, right) => left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0); + return entries; + } + _codecEntriesUnsorted() { + const entries = []; + for (const [peer, counter] of this.#values) { + if (counter > 0) + entries.push({ peer, counter }); + } + return entries; + } + publicEntries() { + return this.codecEntries().map(formatOpId); + } + set(peer, counter) { + if (!Number.isSafeInteger(counter) || counter < 0 || counter > 2147483647) { + throw new RangeError(`version counter is out of range: ${counter}`); + } + const parsed = parsePeerId(peer); + if (counter === 0) + this.#values.delete(parsed); + else + this.#values.set(parsed, counter); + } + merge(other) { + for (const [peer, counter] of other.#values) { + if (counter > (this.#values.get(peer) ?? 0)) + this.#values.set(peer, counter); + } + } +} diff --git a/loro-js/tests/change-block-codec.test.js b/loro-js/tests/change-block-codec.test.js new file mode 100644 index 000000000..beb185ebf --- /dev/null +++ b/loro-js/tests/change-block-codec.test.js @@ -0,0 +1,133 @@ +import { describe, expect, test } from "vitest"; +import { ContainerType, decodeChangeBlock, encodeChangeBlock, validateChangeBlock, } from "../src/codec/index"; +describe("semantic change block codec", () => { + test("round trips a map insert block", () => { + const container = { + kind: "root", + name: "root", + containerType: ContainerType.Map, + }; + const block = { + peers: [0x0102030405060708n], + keys: ["root", "a"], + containers: [container], + positions: [], + changes: [ + { + id: { peer: 0x0102030405060708n, counter: 10 }, + timestamp: 1234n, + dependencies: [], + lamport: 100, + message: undefined, + operations: [ + { + container, + counter: 10, + length: 1, + content: { + type: "map-insert", + key: "a", + value: { type: "i64", value: 10n }, + }, + }, + ], + }, + ], + }; + const encoded = encodeChangeBlock(block); + expect(decodeChangeBlock(encoded)).toEqual(block); + expect(validateChangeBlock(encoded)).toEqual({ + peer: 0x0102030405060708n, + counterStart: 10, + counterEnd: 11, + }); + }); + test("round trips deletes, movable-list operations and tree operations", () => { + const peer = 7n; + const list = { + kind: "root", + name: "list", + containerType: ContainerType.MovableList, + }; + const tree = { + kind: "root", + name: "tree", + containerType: ContainerType.Tree, + }; + const block = { + peers: [peer], + keys: ["list", "tree"], + containers: [list, tree], + positions: [Uint8Array.of(1, 2)], + changes: [ + { + id: { peer, counter: 0 }, + timestamp: 0n, + dependencies: [], + lamport: 0, + message: "ops", + operations: [ + { + container: list, + counter: 0, + length: 1, + content: { + type: "movable-list-insert", + position: 0, + values: [{ type: "string", value: "x" }], + }, + }, + { + container: list, + counter: 1, + length: 1, + content: { + type: "movable-list-delete", + position: 0, + length: 1n, + startId: { peer, counter: 0 }, + }, + }, + { + container: tree, + counter: 2, + length: 1, + content: { + type: "tree-create", + subject: { peer, counter: 2 }, + parent: undefined, + position: Uint8Array.of(1, 2), + }, + }, + { + container: tree, + counter: 3, + length: 1, + content: { + type: "tree-create", + subject: { peer, counter: 3 }, + parent: { peer, counter: 2 }, + position: Uint8Array.of(1, 2), + }, + }, + { + container: tree, + counter: 4, + length: 1, + content: { type: "tree-delete", subject: { peer, counter: 2 } }, + }, + ], + }, + ], + }; + const encoded = encodeChangeBlock(block); + const decoded = decodeChangeBlock(encoded); + expect(decoded.changes).toEqual(block.changes); + expect(decoded.peers).toEqual(expect.arrayContaining([peer, 0xffffffffffffffffn])); + expect(validateChangeBlock(encoded)).toEqual({ + peer, + counterStart: 0, + counterEnd: 5, + }); + }); +}); diff --git a/loro-js/tests/change-block-tables.test.js b/loro-js/tests/change-block-tables.test.js new file mode 100644 index 000000000..f19981f95 --- /dev/null +++ b/loro-js/tests/change-block-tables.test.js @@ -0,0 +1,69 @@ +import { describe, expect, test } from "vitest"; +import { ContainerType, decodeChangeKeys, decodeChangesHeader, decodeChangesMetadata, decodeContainerArena, decodeDeleteStartIds, decodeEncodedOperations, decodePositionArena, encodeChangeKeys, encodeChangesHeader, encodeChangesMetadata, encodeContainerArena, encodeDeleteStartIds, encodeEncodedOperations, encodePositionArena, } from "../src/codec/index"; +describe("change block tables", () => { + test("round trips a change header and metadata", () => { + const header = { + peer: 7n, + peers: [7n, 9n], + counters: [10, 12, 15], + lengths: [2, 3], + lamports: [20, 23], + dependencies: [ + [{ peer: 9n, counter: 3 }], + [ + { peer: 7n, counter: 11 }, + { peer: 9n, counter: 4 }, + ], + ], + }; + const encodedHeader = encodeChangesHeader(header); + expect(decodeChangesHeader(encodedHeader, { + changeCount: 2, + counterStart: 10, + counterLength: 5, + lamportStart: 20, + lamportLength: 6, + })).toEqual(header); + const metadata = { + timestamps: [-1n, 42n], + commitMessages: [undefined, "commit 😀"], + }; + expect(decodeChangesMetadata(encodeChangesMetadata(metadata), 2)).toEqual(metadata); + }); + test("round trips keys and the container arena", () => { + const keys = ["map", "key 😀"]; + const containers = [ + { kind: "root", name: "map", containerType: ContainerType.Map }, + { + kind: "normal", + peer: 9n, + counter: 4, + containerType: ContainerType.Text, + }, + ]; + expect(decodeChangeKeys(encodeChangeKeys(keys))).toEqual(keys); + expect(decodeContainerArena(encodeContainerArena(containers, [7n, 9n], keys), [7n, 9n], keys)).toEqual(containers); + }); + test("round trips operation and delete tables", () => { + const operations = [ + { containerIndex: 0, property: 2, valueType: 11, length: 1 }, + { containerIndex: 1, property: -1, valueType: 9, length: 3 }, + ]; + const deletes = [ + { peerIndex: 1n, counter: 4, length: -3n }, + ]; + expect(decodeEncodedOperations(encodeEncodedOperations(operations))).toEqual(operations); + expect(decodeDeleteStartIds(encodeDeleteStartIds(deletes))).toEqual(deletes); + }); + test("round trips the prefix-compressed position arena", () => { + const positions = [ + Uint8Array.of(1, 2, 3), + Uint8Array.of(1, 2, 4), + Uint8Array.of(1, 2, 4, 255), + Uint8Array.of(1), + new Uint8Array(), + ]; + expect(decodePositionArena(encodePositionArena(positions))).toEqual(positions); + expect(decodePositionArena(new Uint8Array())).toEqual([]); + }); +}); diff --git a/loro-js/tests/change-value.test.js b/loro-js/tests/change-value.test.js new file mode 100644 index 000000000..39f2fd03a --- /dev/null +++ b/loro-js/tests/change-value.test.js @@ -0,0 +1,106 @@ +import { describe, expect, test } from "vitest"; +import { bytesToHex, decodeChangeValue, encodeChangeValue } from "../src/codec/index"; +const bytes = (hex) => Uint8Array.from(hex.match(/../g) ?? [], (byte) => Number.parseInt(byte, 16)); +describe("change value codec", () => { + test.each([ + ["00", { type: "null" }], + ["01", { type: "bool", value: true }], + ["02", { type: "bool", value: false }], + ["037f", { type: "i64", value: -1n }], + ["0a7f", { type: "delta-int", value: -1 }], + ["05026869", { type: "string", value: "hi" }], + ["0603010203", { type: "binary", value: Uint8Array.of(1, 2, 3) }], + ["07ac02", { type: "container-index", value: 300n }], + ["08", { type: "delete-once" }], + ["09", { type: "delete-sequence" }], + ["043ff0000000000000", { type: "double", value: 1 }], + [ + "0b07020301050178", + { + type: "loro-value", + value: { + type: "list", + value: [ + { type: "i64", value: 1n }, + { type: "string", value: "x" }, + ], + }, + }, + ], + [ + "0b08010000", + { + type: "loro-value", + value: { type: "map", value: [[0n, { type: "null" }]] }, + }, + ], + [ + "0c84010001", + { + type: "mark-start", + info: 0x84, + length: 1n, + keyIndex: 0n, + value: { type: "bool", value: true }, + }, + ], + [ + "0d010102", + { + type: "tree-move", + targetIndex: 1n, + parentIsNull: true, + position: 2n, + parentIndex: undefined, + }, + ], + [ + "0d01000203", + { + type: "tree-move", + targetIndex: 1n, + parentIsNull: false, + position: 2n, + parentIndex: 3n, + }, + ], + ["0e010203", { type: "list-move", from: 1n, fromPeerIndex: 2n, lamport: 3n }], + [ + "0f072a0301", + { + type: "list-set", + peerIndex: 7n, + lamport: 42, + value: { type: "i64", value: 1n }, + }, + ], + [ + "1001020301", + { + type: "raw-tree-move", + subjectPeerIndex: 1n, + subjectCounter: 2, + positionIndex: 3n, + parentIsNull: true, + parentPeerIndex: 0n, + parentCounter: 0, + }, + ], + [ + "10010203000405", + { + type: "raw-tree-move", + subjectPeerIndex: 1n, + subjectCounter: 2, + positionIndex: 3n, + parentIsNull: false, + parentPeerIndex: 4n, + parentCounter: 5, + }, + ], + ["910158", { type: "future", tag: 0x91, data: Uint8Array.of(0x58) }], + ])("matches the known bytes %s", (hex, value) => { + expect(decodeChangeValue(bytes(hex))).toEqual(value); + expect(bytesToHex(encodeChangeValue(value))).toBe(hex); + }); +}); diff --git a/loro-js/tests/document.test.js b/loro-js/tests/document.test.js new file mode 100644 index 000000000..8d9eeb7d1 --- /dev/null +++ b/loro-js/tests/document.test.js @@ -0,0 +1,40 @@ +import { describe, expect, test } from "vitest"; +import { EncodeMode, LoroDecodeError, bytesToHex, decodeDocument, decodeFastSnapshot, decodeFastUpdates, encodeDocument, encodeFastSnapshot, encodeFastUpdates, } from "../src/codec/index"; +describe("Loro document envelope", () => { + test("covers mode and body with the checksum", () => { + const encoded = encodeDocument(EncodeMode.FastUpdates, Uint8Array.of(1, 2, 3)); + expect(bytesToHex(encoded.subarray(0, 4))).toBe("6c6f726f"); + expect(encoded.subarray(4, 16)).toEqual(new Uint8Array(12)); + expect(encoded.subarray(20, 22)).toEqual(Uint8Array.of(0, 4)); + expect(decodeDocument(encoded)).toMatchObject({ mode: EncodeMode.FastUpdates }); + const corrupted = encoded.slice(); + corrupted[20] = corrupted[20] ^ 1; + expect(() => decodeDocument(corrupted)).toThrow(LoroDecodeError); + }); +}); +describe("FastSnapshot", () => { + test("round trips all three exact sections", () => { + const snapshot = { + oplog: Uint8Array.of(1, 2), + state: Uint8Array.of(3), + shallowRootState: Uint8Array.of(4, 5, 6), + }; + expect(decodeFastSnapshot(encodeFastSnapshot(snapshot))).toEqual(snapshot); + }); + test("rejects trailing section bytes", () => { + const encoded = encodeFastSnapshot({ + oplog: new Uint8Array(), + state: new Uint8Array(), + shallowRootState: new Uint8Array(), + }); + const document = decodeDocument(encoded); + const withTrailing = encodeDocument(EncodeMode.FastSnapshot, new Uint8Array([...document.body, 0])); + expect(() => decodeFastSnapshot(withTrailing)).toThrow("trailing FastSnapshot bytes"); + }); +}); +describe("FastUpdates", () => { + test("round trips length-prefixed postcard blocks", () => { + const blocks = [new Uint8Array(), Uint8Array.of(1), new Uint8Array(300).fill(7)]; + expect(decodeFastUpdates(encodeFastUpdates(blocks))).toEqual(blocks); + }); +}); diff --git a/loro-js/tests/event-diff.test.js b/loro-js/tests/event-diff.test.js new file mode 100644 index 000000000..56a715ce5 --- /dev/null +++ b/loro-js/tests/event-diff.test.js @@ -0,0 +1,290 @@ +import { describe, expect, test } from "vitest"; +import { SequenceEventDiff } from "../src/runtime/event-diff"; +import { LoroDoc } from "../src/runtime"; +describe("incremental event diffs", () => { + test("composes randomized text edits against a string model", () => { + const original = "abcdefghijklmnopqrstuvwxyz"; + const diff = new SequenceEventDiff("text", original.length); + let current = original; + let random = 1367130551; + const next = () => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return random >>> 0; + }; + for (let step = 0; step < 1000; step += 1) { + if (current.length === 0 || (next() & 1) === 0) { + const position = next() % (current.length + 1); + const value = String.fromCharCode(65 + (next() % 26)); + diff.insertText(position, value); + current = current.slice(0, position) + value + current.slice(position); + } + else { + const position = next() % current.length; + const length = 1 + (next() % Math.min(4, current.length - position)); + diff.delete(position, length); + current = current.slice(0, position) + current.slice(position + length); + } + } + const result = diff.toDiff(); + expect(result.type).toBe("text"); + expect(applyTextDelta(original, result.diff)).toBe(current); + }); + test("composes randomized list edits against an array model", () => { + const original = Array.from({ length: 32 }, (_, index) => index); + const diff = new SequenceEventDiff("list", original.length); + let current = [...original]; + let random = 429261588; + const next = () => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return random >>> 0; + }; + for (let step = 0; step < 1000; step += 1) { + if (current.length === 0 || (next() & 1) === 0) { + const position = next() % (current.length + 1); + const value = `v${step}`; + diff.insertList(position, [value]); + current.splice(position, 0, value); + } + else { + const position = next() % current.length; + const length = 1 + (next() % Math.min(4, current.length - position)); + diff.delete(position, length); + current.splice(position, length); + } + } + const result = diff.toDiff(); + expect(result.type).toBe("list"); + expect(applyListDelta(original, result.diff)).toEqual(current); + }); + test("keeps text formatting as an attributed retain", () => { + const diff = new SequenceEventDiff("text", 6); + diff.formatText(1, 3, "bold", true); + expect(diff.toDiff()).toEqual({ + type: "text", + diff: [{ retain: 1 }, { retain: 3, attributes: { bold: true } }], + }); + }); + test("emits operation-composed local diffs for every indexed container", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const text = source.getText("text"); + const list = source.getList("list"); + const movable = source.getMovableList("movable"); + const map = source.getMap("map"); + const counter = source.getCounter("counter"); + text.insert(0, "abcdef"); + list.push("a"); + list.push("b"); + list.push("c"); + movable.push("a"); + movable.push("b"); + movable.push("c"); + map.set("value", 1); + counter.increment(2); + source.commit(); + const target = new LoroDoc(); + target.setPeerId(2); + target.getText("text").insert(0, "abcdef"); + target.getList("list").push("a"); + target.getList("list").push("b"); + target.getList("list").push("c"); + target.getMovableList("movable").push("a"); + target.getMovableList("movable").push("b"); + target.getMovableList("movable").push("c"); + target.getMap("map").set("value", 1); + target.getCounter("counter").increment(2); + target.commit(); + let batch; + source.subscribe((event) => { + batch = event; + }); + text.delete(1, 2); + text.insert(2, "XY"); + text.mark({ start: 1, end: 4 }, "bold", true); + list.delete(1, 1); + list.insert(1, "B"); + movable.move(0, 2); + movable.set(1, "B"); + map.set("value", 2); + map.set("transient", true); + map.delete("transient"); + counter.increment(3); + counter.decrement(1); + source.commit(); + expect(batch).toBeDefined(); + target.applyDiff(batch.events.map(({ target: id, diff: eventDiff }) => [id, eventDiff])); + expect(target.toJSON()).toEqual(source.toJSON()); + expect(target.getText("text").toDelta()).toEqual(text.toDelta()); + expect(batch.events.find(({ target: id }) => id === map.id)?.diff).toEqual({ + type: "map", + updated: { value: 2 }, + }); + expect(batch.events.find(({ target: id }) => id === counter.id)?.diff).toEqual({ + type: "counter", + increment: 2, + }); + }); + test("records imported concurrent edits at their actual visible positions", () => { + const base = new LoroDoc(); + base.setPeerId(1); + base.getText("text").insert(0, "abcd"); + for (const value of ["a", "b", "c", "d"]) + base.getList("list").push(value); + for (const value of ["a", "b", "c"]) { + base.getMovableList("movable").push(value); + } + base.getMap("map").set("value", 1); + base.getCounter("counter").increment(1); + base.commit(); + const baseVersion = base.oplogVersion(); + const left = base.fork(); + left.setPeerId(2); + left.getText("text").insert(2, "L"); + left.getText("text").delete(0, 1); + left.getText("text").mark({ start: 0, end: 3 }, "bold", true); + left.getList("list").insert(2, "L"); + left.getList("list").delete(0, 1); + left.getMovableList("movable").move(0, 2); + left.getMovableList("movable").set(1, "B"); + left.getMap("map").set("value", 2); + left.getCounter("counter").increment(2); + left.commit(); + const update = left.export({ mode: "update", from: baseVersion }); + const right = base.fork(); + right.setPeerId(3); + right.getText("text").insert(2, "R"); + right.getList("list").insert(2, "R"); + right.getMovableList("movable").insert(1, "R"); + right.commit(); + const mirror = right.fork(); + let batch; + right.subscribe((event) => { + batch = event; + }); + right.import(update); + expect(batch).toBeDefined(); + mirror.applyDiff(batch.events.map(({ target: id, diff: eventDiff }) => [id, eventDiff])); + expect(mirror.toJSON()).toEqual(right.toJSON()); + expect(mirror.getText("text").toDelta()).toEqual(right.getText("text").toDelta()); + }); + test("emits only directly changed tree nodes", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + tree.createNode(); + tree.createNode(); + tree.createNode(); + doc.commit(); + let batch; + doc.subscribe((event) => { + batch = event; + }); + const created = tree.createNode(undefined, 0); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ + type: "tree", + diff: [ + { + target: created.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: created.fractionalIndex(), + }, + ], + }); + const moved = tree.roots()[2]; + const oldIndex = moved.index(); + moved.move(undefined, 0); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ + type: "tree", + diff: [ + { + target: moved.id, + action: "move", + parent: undefined, + index: 0, + fractionalIndex: moved.fractionalIndex(), + oldParent: undefined, + oldIndex, + }, + ], + }); + }); + test("reports local tree reindexing during a concurrent import", () => { + const incoming = new LoroDoc(); + incoming.setPeerId(1); + const incomingNode = incoming.getTree("tree").createNode(); + incoming.commit(); + const target = new LoroDoc(); + target.setPeerId(2); + const localNode = target.getTree("tree").createNode(); + target.commit(); + let batch; + target.subscribe((event) => { + batch = event; + }); + target.import(incoming.export({ mode: "update" })); + expect(batch?.events[0]?.diff).toEqual({ + type: "tree", + diff: [ + { + target: localNode.id, + action: "move", + parent: undefined, + index: 1, + fractionalIndex: localNode.fractionalIndex(), + oldParent: undefined, + oldIndex: 0, + }, + { + target: incomingNode.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: incomingNode.fractionalIndex(), + }, + ], + }); + }); +}); +function applyTextDelta(original, delta) { + let cursor = 0; + let output = ""; + for (const operation of delta) { + if ("retain" in operation) { + output += original.slice(cursor, cursor + operation.retain); + cursor += operation.retain; + } + else if ("delete" in operation) { + cursor += operation.delete; + } + else { + output += operation.insert; + } + } + return output + original.slice(cursor); +} +function applyListDelta(original, delta) { + let cursor = 0; + const output = []; + for (const operation of delta) { + if ("retain" in operation) { + output.push(...original.slice(cursor, cursor + operation.retain)); + cursor += operation.retain; + } + else if ("delete" in operation) { + cursor += operation.delete; + } + else { + output.push(...operation.insert); + } + } + output.push(...original.slice(cursor)); + return output; +} diff --git a/loro-js/tests/fugue-index.test.js b/loro-js/tests/fugue-index.test.js new file mode 100644 index 000000000..c616b5bab --- /dev/null +++ b/loro-js/tests/fugue-index.test.js @@ -0,0 +1,21 @@ +import { describe, expect, test, vi } from "vitest"; +import { LoroText } from "../src/index"; +import { SequenceIndex } from "../src/runtime/sequence-index"; +describe("Fugue origin index", () => { + test("does not probe every descendant in a concurrent text run", () => { + const probeCount = (length) => { + const text = new LoroText(); + text._insertFugue(0, "a".repeat(length), { peer: 1n, counter: 0 }, 0, new Map()); + const atPhysical = vi.spyOn(SequenceIndex.prototype, "atPhysicalRaw"); + text._insertFugue(0, "b", { peer: 2n, counter: 0 }, 0, new Map()); + const probes = atPhysical.mock.calls.length; + atPhysical.mockRestore(); + expect(text.toString()).toBe(`${"a".repeat(length)}b`); + return probes; + }; + const shortRunProbes = probeCount(128); + const longRunProbes = probeCount(16384); + expect(longRunProbes).toBeLessThanOrEqual(shortRunProbes + 2); + expect(longRunProbes).toBeLessThan(8); + }); +}); diff --git a/loro-js/tests/indexes.test.js b/loro-js/tests/indexes.test.js new file mode 100644 index 000000000..1ff7fd377 --- /dev/null +++ b/loro-js/tests/indexes.test.js @@ -0,0 +1,620 @@ +import { describe, expect, test } from "vitest"; +import { OrderedIndex } from "../src/runtime/ordered-index"; +import { SequenceDeletionIndex } from "../src/runtime/sequence-deletion-index"; +import { SequenceIndex } from "../src/runtime/sequence-index"; +import { TextStyleIndex } from "../src/runtime/text-style-index"; +describe("runtime indexes", () => { + test("maintains sequence ranks, metrics, and id lookups", () => { + const sequence = new SequenceIndex((element) => ({ + utf16: element.value.length, + utf8: new TextEncoder().encode(element.value).length, + })); + const physical = []; + let counter = 0; + let random = 305419896; + const nextRandom = () => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return random >>> 0; + }; + const visible = () => physical.filter((element) => !element.deleted); + for (let step = 0; step < 2000; step += 1) { + const current = visible(); + const operation = current.length === 0 ? 0 : nextRandom() % 3; + if (operation === 0) { + const position = nextRandom() % (current.length + 1); + const value = ["a", "文", "🙂"][nextRandom() % 3]; + const element = { + id: { peer: 1n, counter: counter++ }, + deleted: false, + deletedBy: [], + value, + }; + const physicalPosition = position === current.length + ? physical.length + : physical.indexOf(current[position]); + physical.splice(physicalPosition, 0, element); + sequence.insertAtVisible(position, [element]); + } + else if (operation === 1) { + const element = current[nextRandom() % current.length]; + sequence.setDeleted(element, true); + } + else { + const from = nextRandom() % current.length; + const to = nextRandom() % current.length; + if (from !== to) { + const element = current[from]; + const physicalPosition = physical.indexOf(element); + physical.splice(physicalPosition, 1); + const afterRemoval = visible(); + const destination = to >= afterRemoval.length + ? physical.length + : physical.indexOf(afterRemoval[to]); + physical.splice(destination, 0, element); + sequence.moveVisible(from, to); + } + } + if (step % 25 !== 0) + continue; + const expectedVisible = visible(); + expect(sequence.all()).toEqual(physical); + expect(sequence.visible()).toEqual(expectedVisible); + expect(sequence.allLength).toBe(physical.length); + expect(sequence.visibleLength).toBe(expectedVisible.length); + expect(sequence.visibleUtf16Length).toBe(expectedVisible.reduce((length, element) => length + element.value.length, 0)); + expect(sequence.visibleUtf8Length).toBe(new TextEncoder().encode(expectedVisible.map(({ value }) => value).join("")) + .length); + for (const [index, element] of physical.entries()) { + expect(sequence.atPhysical(index)).toBe(element); + expect(sequence.physicalIndexOf(element)).toBe(index); + expect(sequence.findById(element.id)).toBe(element); + } + for (const [index, element] of expectedVisible.entries()) { + expect(sequence.atVisible(index)).toBe(element); + expect(sequence.visibleIndexOf(element)).toBe(index); + } + const start = expectedVisible.length >>> 2; + const end = expectedVisible.length - start; + expect(sequence.visibleRange(start, end)).toEqual(expectedVisible.slice(start, end)); + expect(sequence.visibleIdRuns(start, end)).toEqual(compressIdRuns(expectedVisible.slice(start, end))); + } + }); + test("stops visible range predicates after the first match", () => { + const sequence = new SequenceIndex(); + sequence.insertAtVisible(0, Array.from({ length: 96 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: counter % 7 === 0, + deletedBy: [], + value: String(counter), + }))); + let visits = 0; + expect(sequence.someVisibleRange(10, 80, (element) => { + visits += 1; + return element.value === "18"; + })).toBe(true); + expect(visits).toBeLessThan(10); + visits = 0; + expect(sequence.someVisibleRange(10, 20, () => { + visits += 1; + return false; + })).toBe(false); + expect(visits).toBe(10); + }); + test("visits raw visible storage ranges and locates insertion neighbor ids", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 6 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtVisible(0, elements); + sequence.setDeleted(elements[1], true); + sequence.setDeleted(elements[4], true); + const ranges = []; + sequence.forEachVisibleStorageRange((_storage, start, end) => { + ranges.push({ start, end }); + }); + expect(ranges).toEqual([ + { start: 0, end: 1 }, + { start: 2, end: 4 }, + { start: 5, end: 6 }, + ]); + const context = { + leftPeer: undefined, + leftCounter: 0, + startIndex: 0, + rightPeer: undefined, + rightCounter: 0, + }; + const locate = (position) => ({ + ...sequence.visibleInsertionIdContext(position, context), + }); + expect(locate(0)).toEqual({ + leftPeer: undefined, + leftCounter: 0, + startIndex: 0, + rightPeer: 1n, + rightCounter: 0, + }); + expect(locate(1)).toEqual({ + leftPeer: 1n, + leftCounter: 0, + startIndex: 1, + rightPeer: 1n, + rightCounter: 1, + }); + expect(locate(2)).toEqual({ + leftPeer: 1n, + leftCounter: 2, + startIndex: 3, + rightPeer: 1n, + rightCounter: 3, + }); + expect(locate(4)).toEqual({ + leftPeer: 1n, + leftCounter: 5, + startIndex: 6, + rightPeer: undefined, + rightCounter: 0, + }); + expect(() => sequence.loadValidatedSpans([])).toThrow(/validated sequence spans require an empty index/u); + }); + test("maps id runs to visible metric ranges without crossing inserts", () => { + const sequence = new SequenceIndex((element) => ({ + utf16: element.value.length, + utf8: new TextEncoder().encode(element.value).length, + })); + const original = ["a", "🙂", "b", "c", "d"].map((value, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value, + })); + sequence.insertAtVisible(0, original); + sequence.insertAtVisible(2, [ + { + id: { peer: 2n, counter: 0 }, + deleted: false, + deletedBy: [], + value: "X", + }, + ]); + sequence.setDeleted(original[3], true); + expect(sequence.visibleMetricRangesForIdRuns([{ start: { peer: 1n, counter: 0 }, length: 5 }], "utf16")).toEqual([ + { start: 0, end: 3 }, + { start: 4, end: 6 }, + ]); + }); + test("maintains ordered ranks through insertion and deletion", () => { + const index = new OrderedIndex((left, right) => left.key - right.key); + const values = Array.from({ length: 1000 }, (_, key) => ({ key })); + for (let offset = 0; offset < 10; offset += 1) { + for (let key = offset; key < values.length; key += 10) + index.add(values[key]); + } + expect(index.values()).toEqual(values); + for (const [position, value] of values.entries()) { + expect(index.at(position)).toBe(value); + expect(index.indexOf(value)).toBe(position); + } + for (const value of values.filter(({ key }) => key % 3 === 0)) + index.delete(value); + const remaining = values.filter(({ key }) => key % 3 !== 0); + expect(index.values()).toEqual(remaining); + for (const [position, value] of remaining.entries()) { + expect(index.indexOf(value)).toBe(position); + } + }); + test("indexes text styles by id runs and causal version", () => { + const styles = new TextStyleIndex(); + const outer = { + startId: { peer: 7n, counter: 0 }, + lamport: 1, + info: 0, + value: true, + }; + const inner = { + startId: { peer: 8n, counter: 0 }, + lamport: 2, + info: 0, + value: null, + }; + styles.add([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", outer); + styles.add([{ start: { peer: 1n, counter: 25 }, length: 50 }], "bold", inner); + expect(styles.metasAt({ peer: 1n, counter: 10 }).get("bold")).toBe(outer); + expect(styles.metasAt({ peer: 1n, counter: 50 }).get("bold")).toBe(inner); + expect(styles.metasAt({ peer: 1n, counter: 50 }, new Map([[7n, 1]])).get("bold")).toBe(outer); + expect(styles.rangeHasKey([{ start: { peer: 1n, counter: 30 }, length: 20 }], "bold")).toBe(false); + expect(styles.runsContainMeta([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", outer.startId)).toBe(true); + expect(styles.runsContainMeta([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", inner.startId)).toBe(false); + expect(styles.transitions([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", new Map([[7n, 1]]), undefined)).toEqual([ + { + run: { start: { peer: 1n, counter: 25 }, length: 50 }, + before: outer, + after: inner, + }, + ]); + }); + test("indexes forward and reverse deletion runs in both directions", () => { + const deletions = new SequenceDeletionIndex(); + deletions.add({ peer: 1n, counter: 10 }, 5, { peer: 2n, counter: 100 }, 1); + deletions.add({ peer: 1n, counter: 12 }, 4, { peer: 3n, counter: 200 }, -1); + expect(deletions.deletionIdsAt({ peer: 1n, counter: 13 })).toEqual([ + { peer: 2n, counter: 103 }, + { peer: 3n, counter: 202 }, + ]); + expect(deletions.someDeletion({ peer: 1n, counter: 15 }, (id) => id.counter < 201)).toBe(true); + expect(deletions.everyDeletion({ peer: 1n, counter: 13 }, (id) => id.counter >= 100)).toBe(true); + expect(deletions.targetRunsDeletedBy(2n, 101, 104)).toEqual([ + { start: { peer: 1n, counter: 11 }, length: 3 }, + ]); + expect(deletions.targetRunsDeletedBy(3n, 201, 204)).toEqual([ + { start: { peer: 1n, counter: 12 }, length: 3 }, + ]); + }); + test("indexes visibility at an earlier causal version", () => { + const sequence = new SequenceIndex(); + const elements = [ + { id: { peer: 1n, counter: 0 }, deleted: false, deletedBy: [], value: "a" }, + { id: { peer: 2n, counter: 0 }, deleted: false, deletedBy: [], value: "b" }, + { id: { peer: 1n, counter: 1 }, deleted: false, deletedBy: [], value: "c" }, + ]; + sequence.insertAtPhysical(0, elements); + sequence.setDeleted(elements[0], true); + sequence.addDeletion(elements[0], { peer: 2n, counter: 1 }); + const beforeConcurrentPeer = sequence.causalView(new Map([ + [1n, 2], + [2n, 0], + ])); + expect(beforeConcurrentPeer.length).toBe(2); + expect(beforeConcurrentPeer.range(0, 2).map(({ value }) => value)).toEqual([ + "a", + "c", + ]); + expect(beforeConcurrentPeer.at(1)?.value).toBe("c"); + const mutableVersion = new Map([ + [1n, 2], + [2n, 0], + ]); + const immutableView = sequence.causalView(mutableVersion); + mutableVersion.set(2n, 2); + expect(immutableView.range(0, immutableView.length).map(({ value }) => value)).toEqual(["a", "c"]); + expect(sequence.causalView(new Map([ + [1n, 2], + [2n, 0], + ]))).toBe(beforeConcurrentPeer); + const withoutPeerOneTail = sequence.causalView(new Map([ + [1n, 1], + [2n, 2], + ])); + expect(withoutPeerOneTail.range(0, withoutPeerOneTail.length).map(({ value }) => value)).toEqual(["b"]); + expect(sequence.causalView(new Map([ + [1n, 2], + [2n, 0], + ]))).toBe(beforeConcurrentPeer); + const latest = sequence.causalView(new Map([ + [1n, 2], + [2n, 2], + ])); + expect(latest.range(0, latest.length).map(({ value }) => value)).toEqual(["b", "c"]); + sequence.moveBefore(elements[2], elements[0]); + const afterMove = sequence.causalView(new Map([ + [1n, 2], + [2n, 0], + ])); + expect(afterMove).not.toBe(beforeConcurrentPeer); + expect(afterMove.range(0, afterMove.length).map(({ value }) => value)).toEqual([ + "c", + "a", + ]); + }); + test("packs batch inserts into bounded sequence spans", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 10000 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: "x", + })); + sequence.insertAtPhysical(0, elements); + expect(sequence.spanCount).toBeLessThanOrEqual(Math.ceil(elements.length / 32)); + for (let index = 0; index < elements.length; index += 97) { + sequence.setDeleted(elements[index], true); + } + expect(sequence.spanCount).toBeLessThanOrEqual(Math.ceil(elements.length / 32)); + expect(sequence.visibleLength).toBe(elements.filter((element) => !element.deleted).length); + }); + test("skips deleted subtrees when finding visible neighbors", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 10000 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: "x", + })); + sequence.insertAtPhysical(0, elements); + sequence.deleteIdSpan({ peer: 1n, counter: 1 }, elements.length - 2, { + peer: 2n, + counter: 0, + }); + expect(sequence.nextVisible(elements[0])).toBe(elements.at(-1)); + expect(sequence.previousVisible(elements.at(-1))).toBe(elements[0]); + expect(sequence.nextVisible(elements[elements.length >>> 1])).toBe(elements.at(-1)); + expect(sequence.previousVisible(elements[elements.length >>> 1])).toBe(elements[0]); + }); + test("materializes lazy span deletions before restoring or restructuring", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 10000 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtPhysical(0, elements); + sequence.deleteIdSpan({ peer: 1n, counter: 0 }, elements.length, { + peer: 2n, + counter: 0, + }); + sequence.setDeleted(elements[5000], false); + expect(sequence.visible()).toEqual([elements[5000]]); + expect(sequence.visibleIndexOf(elements[5000])).toBe(0); + expect(sequence.atPhysical(4999)?.deleted).toBe(true); + const inserted = { + id: { peer: 3n, counter: 0 }, + deleted: false, + deletedBy: [], + value: "inserted", + }; + sequence.insertAtPhysical(5000, [inserted]); + expect(sequence.visible()).toEqual([inserted, elements[5000]]); + expect(sequence.all().filter((element) => !element.deleted)).toEqual([ + inserted, + elements[5000], + ]); + }); + test("keeps rank and range queries correct after lazy range restoration", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 10000 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtPhysical(0, elements); + const all = [{ start: { peer: 1n, counter: 0 }, length: elements.length }]; + sequence.setIdRunsDeleted(all); + sequence.setIdRunsVisible([{ start: { peer: 1n, counter: 100 }, length: 9800 }]); + expect(sequence.visibleLength).toBe(9800); + expect(sequence.atVisible(0)).toBe(elements[100]); + expect(sequence.atVisible(5000)).toBe(elements[5100]); + expect(sequence.atVisible(9799)).toBe(elements[9899]); + expect(sequence.visibleIndexOf(elements[5100])).toBe(5000); + expect(sequence.metricOffsetAtVisibleIndex(5000, "utf16")).toBe(5000); + expect(sequence.visibleIndexAtMetricOffset(5000, "utf16")).toBe(5000); + expect(sequence.visibleIdRuns(123, 9123)).toEqual([ + { start: { peer: 1n, counter: 223 }, length: 9000 }, + ]); + expect(sequence.visibleRange(4999, 5002)).toEqual([ + elements[5099], + elements[5100], + elements[5101], + ]); + expect(sequence.previousVisible(elements[100])).toBeUndefined(); + expect(sequence.nextVisible(elements[9899])).toBeUndefined(); + }); + test("deletes id spans while preserving causal delete order", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 96 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtPhysical(0, elements); + sequence.deleteIdSpan({ peer: 1n, counter: 20 }, 50, { + peer: 2n, + counter: 100, + }); + expect(sequence.visibleLength).toBe(46); + expect(sequence.visible().map(({ id }) => id.counter)).toEqual([ + ...Array.from({ length: 20 }, (_, counter) => counter), + ...Array.from({ length: 26 }, (_, counter) => counter + 70), + ]); + expect(sequence.elementsDeletedBy(2n, 100, 150).map(({ id }) => id.counter)).toEqual(Array.from({ length: 50 }, (_, counter) => counter + 20)); + const halfway = sequence.causalView(new Map([ + [1n, 96], + [2n, 125], + ])); + expect(halfway.length).toBe(71); + expect(halfway.range(19, 22).map(({ id }) => id.counter)).toEqual([19, 45, 46]); + const reversed = new SequenceIndex(); + const reversedElements = Array.from({ length: 8 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + reversed.insertAtPhysical(0, reversedElements); + reversed.deleteIdSpan({ peer: 1n, counter: 2 }, -4, { + peer: 2n, + counter: 10, + }); + const partialReverse = reversed.causalView(new Map([ + [1n, 8], + [2n, 12], + ])); + expect(partialReverse.range(0, partialReverse.length).map(({ id }) => id.counter)).toEqual([0, 1, 2, 3, 6, 7]); + }); + test("checks scalar and range deletions only on the restored ID runs", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 96 }, (_, counter) => ({ + id: { peer: counter === 95 ? 4n : 1n, counter: counter === 95 ? 0 : counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtPhysical(0, elements); + sequence.deleteIdSpan({ peer: 1n, counter: 0 }, 95, { + peer: 2n, + counter: 0, + }); + sequence.deleteElement(elements[95], { peer: 3n, counter: 0 }); + const restored = [{ start: { peer: 1n, counter: 0 }, length: 95 }]; + expect(sequence.canShowIdRunsAt(restored, new Map([ + [1n, 95], + [2n, 0], + [3n, 1], + [4n, 1], + ]))).toBe(true); + sequence.addDeletion(elements[0], { peer: 3n, counter: 1 }); + expect(sequence.canShowIdRunsAt(restored, new Map([ + [1n, 95], + [2n, 0], + [3n, 2], + [4n, 1], + ]))).toBe(false); + }); + test("coalesces adjacent single-element inserts into bounded spans", () => { + const sequence = new SequenceIndex(); + for (let counter = 0; counter < 10000; counter += 1) { + sequence.insertAtPhysical(sequence.allLength, [ + { + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + }, + ]); + } + expect(sequence.allLength).toBe(10000); + expect(sequence.spanCount).toBe(Math.ceil(10000 / 32)); + for (const counter of [0, 31, 32, 4999, 9999]) { + const element = sequence.findById({ peer: 1n, counter }); + expect(element?.value).toBe(String(counter)); + expect(sequence.physicalIndexOf(element)).toBe(counter); + } + }); + test("ranges across an excluded element", () => { + const sequence = new SequenceIndex(); + const base = Array.from({ length: 6 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String.fromCharCode(97 + counter), + })); + sequence.insertAtPhysical(0, base); + sequence.recordOperationId({ peer: 1n, counter: 6 }); + sequence.recordOperationId({ peer: 1n, counter: 7 }); + sequence.recordOperationId({ peer: 1n, counter: 8 }); + sequence.recordOperationId({ peer: 1n, counter: 9 }); + sequence.insertAtPhysical(3, [ + { + id: { peer: 1n, counter: 10 }, + deleted: false, + deletedBy: [], + value: "X", + }, + ]); + sequence.recordOperationId({ peer: 1n, counter: 11 }); + sequence.recordOperationId({ peer: 1n, counter: 12 }); + const view = sequence.causalView(new Map([[1n, 8]])); + expect(view.length).toBe(6); + expect(view.range(2, 4).map(({ value }) => value)).toEqual(["c", "d"]); + expect(view.idRuns(0, view.length)).toEqual([ + { start: { peer: 1n, counter: 0 }, length: 6 }, + ]); + }); + test("seeks sparse operation counters without walking numeric gaps", () => { + const sequence = new SequenceIndex(); + const first = { + id: { peer: 1n, counter: 0 }, + deleted: false, + deletedBy: [], + value: "a", + }; + const distant = { + id: { peer: 1n, counter: 1000000000 }, + deleted: false, + deletedBy: [], + value: "b", + }; + sequence.insertAtPhysical(0, [first, distant]); + sequence.setDeleted(first, true); + sequence.addDeletion(first, { peer: 2n, counter: 1000000000 }); + const early = sequence.causalView(new Map([ + [1n, 1], + [2n, 0], + ])); + expect(early.range(0, early.length).map(({ value }) => value)).toEqual(["a"]); + expect(sequence.elementsDeletedBy(2n, 999999999, 1000000001)).toEqual([first]); + }); + test("promotes a deletion counter to a set only for one-to-many deletes", () => { + const sequence = new SequenceIndex(); + const elements = Array.from({ length: 3 }, (_, counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtPhysical(0, elements); + for (const element of elements.slice(0, 2)) { + sequence.setDeleted(element, true); + sequence.addDeletion(element, { peer: 2n, counter: 0 }); + } + sequence.addDeletion(elements[0], { peer: 3n, counter: 0 }); + expect(sequence.elementsDeletedBy(2n, 0, 1)).toEqual(elements.slice(0, 2)); + const beforeDelete = sequence.causalView(new Map([ + [1n, 3], + [2n, 0], + [3n, 0], + ])); + expect(beforeDelete.range(0, beforeDelete.length)).toEqual(elements); + const afterSecondDelete = sequence.causalView(new Map([ + [1n, 3], + [2n, 0], + [3n, 1], + ])); + expect(afterSecondDelete.range(0, afterSecondDelete.length)).toEqual(elements.slice(1)); + }); + test("merges out-of-order counter runs", () => { + const sequence = new SequenceIndex(); + const counters = [10, 0, 2, 1, 1000000000, 11]; + const elements = counters.map((counter) => ({ + id: { peer: 1n, counter }, + deleted: false, + deletedBy: [], + value: String(counter), + })); + sequence.insertAtPhysical(0, elements); + const early = sequence.causalView(new Map([[1n, 2]])); + expect(early.range(0, early.length).map(({ id }) => id.counter)).toEqual([0, 1]); + for (let index = 0; index < elements.length; index += 1) { + sequence.setDeleted(elements[index], true); + sequence.addDeletion(elements[index], { + peer: 2n, + counter: counters[index], + }); + } + expect(sequence.elementsDeletedBy(2n, 1, 11).map(({ id }) => id.counter)).toEqual([ + 10, 2, 1, + ]); + }); +}); +function compressIdRuns(elements) { + const runs = []; + for (const { id } of elements) { + const previous = runs.at(-1); + if (previous !== undefined && + previous.start.peer === id.peer && + previous.start.counter + previous.length === id.counter) { + previous.length += 1; + } + else { + runs.push({ start: { ...id }, length: 1 }); + } + } + return runs; +} diff --git a/loro-js/tests/lz4.test.js b/loro-js/tests/lz4.test.js new file mode 100644 index 000000000..a5f60350d --- /dev/null +++ b/loro-js/tests/lz4.test.js @@ -0,0 +1,65 @@ +import { describe, expect, test } from "vitest"; +import { bytesToHex, decodeLz4Frame, encodeLz4FrameRaw, hexToBytes, } from "../src/codec/index"; +describe("LZ4 frame", () => { + test("decodes a compressed overlapping match", () => { + const frame = hexToBytes("04224d18604082040000001061010000000000"); + expect(new TextDecoder().decode(decodeLz4Frame(frame))).toBe("aaaaa"); + }); + test.each([0, 1, 65536, 65537, 300000])("round trips %i canonical frame bytes", (length) => { + const input = new Uint8Array(length); + for (let index = 0; index < input.length; index += 1) { + input[index] = index & 0xff; + } + expect(decodeLz4Frame(encodeLz4FrameRaw(input), { + requireCanonicalProfile: true, + })).toEqual(input); + }); + test("encodes an overlapping match with the required trailing literals", () => { + const input = new TextEncoder().encode("aaaaaaaaaaaaaaaaaa"); + const encoded = encodeLz4FrameRaw(input); + expect(bytesToHex(encoded)).toBe("04224d186040820a0000001861010050616161616100000000"); + expect(decodeLz4Frame(encoded)).toEqual(input); + }); + test("compresses highly repetitive blocks", () => { + const input = new Uint8Array(256 * 1024).fill(0x5a); + const encoded = encodeLz4FrameRaw(input); + expect(readFirstBlockInfo(encoded) & 2147483648).toBe(0); + expect(encoded.length).toBeLessThan(input.length / 100); + expect(decodeLz4Frame(encoded)).toEqual(input); + }); + test("stores incompressible blocks without expansion", () => { + const input = deterministicNoise(32 * 1024); + const encoded = encodeLz4FrameRaw(input); + expect(readFirstBlockInfo(encoded) & 2147483648).not.toBe(0); + expect(encoded.length).toBe(input.length + 15); + expect(decodeLz4Frame(encoded)).toEqual(input); + }); + test("round trips mixed noisy and repeated ranges", () => { + for (const [caseIndex, length] of [ + 13, 14, 15, 31, 255, 256, 4095, 65535, 65536, 65537, + ].entries()) { + const input = deterministicNoise(length, 305419896 + caseIndex); + for (let start = 97; start < input.length; start += 211) { + const copyLength = Math.min(80, input.length - start); + for (let index = 0; index < copyLength; index += 1) { + input[start + index] = input[index % 37]; + } + } + expect(decodeLz4Frame(encodeLz4FrameRaw(input))).toEqual(input); + } + }); +}); +function readFirstBlockInfo(frame) { + return new DataView(frame.buffer, frame.byteOffset + 7, 4).getUint32(0, true); +} +function deterministicNoise(length, seed = 305419896) { + const output = new Uint8Array(length); + let state = seed; + for (let index = 0; index < output.length; index += 1) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + output[index] = state >>> 24; + } + return output; +} diff --git a/loro-js/tests/primitives.test.js b/loro-js/tests/primitives.test.js new file mode 100644 index 000000000..877e239a7 --- /dev/null +++ b/loro-js/tests/primitives.test.js @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { ByteReader, ByteWriter, LORO_XXHASH_SEED, PostcardReader, PostcardWriter, bytesToHex, readSleb128, readUleb128, writeSleb128, writeUleb128, xxhash32, } from "../src/codec/index"; +describe("xxHash32", () => { + test.each([ + [new Uint8Array(), 0, 46947589], + [new TextEncoder().encode("a"), 0, 1426945110], + [new TextEncoder().encode("abc"), 0, 852579327], + [new TextEncoder().encode("message digest"), 0, 2090108052], + [new TextEncoder().encode("abc"), LORO_XXHASH_SEED, 2784525984], + ])("matches the reference vector", (input, seed, expected) => { + expect(xxhash32(input, seed)).toBe(expected); + }); +}); +describe("LEB128", () => { + test.each([0n, 1n, 127n, 128n, 0xffffffffn, 0xffffffffffffffffn])("round trips unsigned %s", (value) => { + const writer = new ByteWriter(); + writeUleb128(writer, value); + const reader = new ByteReader(writer.toUint8Array()); + expect(readUleb128(reader)).toBe(value); + expect(reader.remaining).toBe(0); + }); + test.each([-0x8000000000000000n, -65n, -1n, 0n, 63n, 64n, 0x7fffffffffffffffn])("round trips signed %s", (value) => { + const writer = new ByteWriter(); + writeSleb128(writer, value); + const reader = new ByteReader(writer.toUint8Array()); + expect(readSleb128(reader)).toBe(value); + expect(reader.remaining).toBe(0); + }); + test("uses signed LEB128 rather than postcard zigzag", () => { + const writer = new ByteWriter(); + writeSleb128(writer, -1n); + expect(bytesToHex(writer.toUint8Array())).toBe("7f"); + const postcard = new PostcardWriter(); + postcard.writeI32(-1); + expect(bytesToHex(postcard.toUint8Array())).toBe("01"); + }); +}); +describe("postcard primitives", () => { + test("round trips signed integers, strings, bytes and arrays", () => { + const writer = new PostcardWriter(); + writer.writeI32(-123); + writer.writeI64(-9007199254740993n); + writer.writeString("Loro 😀"); + writer.writeBytes(Uint8Array.of(0, 1, 255)); + writer.writeArray([1, 2, 300], (output, value) => output.writeU32(value)); + const reader = new PostcardReader(writer.toUint8Array()); + expect(reader.readI32()).toBe(-123); + expect(reader.readI64()).toBe(-9007199254740993n); + expect(reader.readString()).toBe("Loro 😀"); + expect(reader.readBytes()).toEqual(Uint8Array.of(0, 1, 255)); + expect(reader.readArray((input) => input.readU32())).toEqual([1, 2, 300]); + reader.assertEnd(); + }); +}); diff --git a/loro-js/tests/runtime.test.js b/loro-js/tests/runtime.test.js new file mode 100644 index 000000000..de7517542 --- /dev/null +++ b/loro-js/tests/runtime.test.js @@ -0,0 +1,1708 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, test } from "vitest"; +import { ContainerType, decodeFastSnapshot, decodePostcardVersionVector, decodeStateSnapshotStore, decodeSstable, encodeFastSnapshot, encodePostcardVersionVector, encodeStateSnapshotStore, encodeSstable, } from "../src/codec"; +import { Awareness, AwarenessWasm, Cursor, decodeFrontiers, decodeImportBlobMeta, encodeFrontiers, EphemeralStore, EphemeralStoreWasm, LORO_VERSION, LoroCounter, LoroDoc, LoroList, LoroMap, LoroMovableList, LoroText, redactJsonUpdates, UndoManager, VersionVector, } from "../src/index"; +const fixture = (name) => new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); +const withDuplicateTextId = (snapshot) => { + const stateStore = decodeStateSnapshotStore(snapshot.state); + if (stateStore.kind !== "sstable") + throw new Error("expected snapshot state"); + let changed = false; + const containers = stateStore.containers.map((entry) => { + const state = entry.wrapper.state; + if (changed || state.kind !== ContainerType.Text) + return entry; + const span = state.spans.find(({ length }) => length >= 2); + if (span === undefined) + return entry; + const peer = state.peers[Number(span.peerIndex)]; + const aliasPeerIndex = BigInt(state.peers.length); + changed = true; + return { + ...entry, + wrapper: { + ...entry.wrapper, + state: { + ...state, + peers: [...state.peers, peer], + spans: state.spans.flatMap((item) => item === span + ? [ + { ...span, length: 1 }, + { ...span, peerIndex: aliasPeerIndex, length: 1 }, + { ...span, counter: span.counter + 2, length: span.length - 2 }, + ].filter(({ length }) => length > 0) + : [item]), + }, + }, + }; + }); + if (!changed) + throw new Error("expected a splittable text span"); + return encodeFastSnapshot({ + ...snapshot, + state: encodeStateSnapshotStore({ ...stateStore, containers }, { compression: "none" }), + }); +}; +describe("loro-wasm-compatible runtime", () => { + test("keeps existing container handlers usable after freeing the doc wrapper", () => { + const doc = new LoroDoc(); + const text = doc.getText("text"); + const list = doc.getList("list"); + const map = doc.getMap("map"); + doc.free(); + text.insert(0, "123"); + list.insert(0, 1); + map.set("key", 8); + expect(text.toString()).toBe("123"); + expect(list.toJSON()).toEqual([1]); + expect(map.toJSON()).toEqual({ key: 8 }); + }); + test("edits map, list, text, movable-list, tree and counter containers", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getMap("map").set("answer", 42); + const list = doc.getList("list"); + list.push("a"); + list.push("b"); + list.delete(0, 1); + const text = doc.getText("text"); + text.insert(0, "a😀b"); + text.delete(0, 1); + const movable = doc.getMovableList("movable"); + movable.push(1); + movable.push(2); + movable.move(0, 1); + movable.set(0, 3); + const tree = doc.getTree("tree"); + const root = tree.createNode(); + root.data.set("title", "root"); + root.createNode(); + doc.getCounter("counter").increment(2.5); + expect(doc.toJSON()).toEqual({ + map: { answer: 42 }, + list: ["b"], + text: "😀b", + movable: [3, 1], + tree: [ + expect.objectContaining({ + id: root.id, + meta: { title: "root" }, + children: [expect.objectContaining({ parent: root.id })], + }), + ], + counter: 2.5, + }); + }); + test("returns wasm-shaped shallow values and deep values with container ids", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const map = doc.getMap("map"); + map.set("plain", 1); + const child = map.setContainer("body", new LoroText()); + child.insert(0, "hello"); + const tree = doc.getTree("tree"); + const node = tree.createNode(); + node.data.set("name", "root"); + expect(map.getShallowValue()).toEqual({ plain: 1, body: child.id }); + expect(doc.getShallowValue()).toEqual({ map: map.id, tree: tree.id }); + expect(tree.getShallowValue()).toEqual([ + { + id: node.id, + parent: null, + index: 0, + fractional_index: "80", + meta: node.data.id, + children: [], + }, + ]); + expect(tree.toJSON()).toEqual([ + { + id: node.id, + parent: null, + index: 0, + fractional_index: "80", + meta: { name: "root" }, + children: [], + }, + ]); + expect(doc.getDeepValueWithID()).toMatchObject({ + map: { + cid: map.id, + value: { + body: { cid: child.id, value: "hello" }, + plain: 1, + }, + }, + }); + }); + test("evaluates JSONPath selectors, recursion, slices, and filters", () => { + const doc = new LoroDoc(); + const store = doc.getMap("store"); + store.set("books", [ + { title: "1984", author: "George Orwell", price: 10, available: true }, + { title: "Animal Farm", author: "George Orwell", price: 8, available: true }, + { title: "Brave New World", author: "Aldous Huxley", price: 12, available: false }, + { title: "Fahrenheit 451", author: "Ray Bradbury", price: 9, available: true }, + { title: "Pride and Prejudice", author: "Jane Austen", price: 7, available: true }, + ]); + store.set("featured_author", "George Orwell"); + store.set("featured_authors", ["George Orwell", "Jane Austen"]); + const body = store.setContainer("body", new LoroText()); + body.insert(0, "content"); + const outline = doc.getTree("outline"); + const firstRoot = outline.createNode(); + firstRoot.data.set("label", "first"); + firstRoot.createNode().data.set("label", "child"); + outline.createNode().data.set("label", "last"); + expect(doc.JSONPath("$['store'].books[0].title")).toEqual(["1984"]); + expect(doc.JSONPath("$.store.books[*].title")).toEqual([ + "1984", + "Animal Farm", + "Brave New World", + "Fahrenheit 451", + "Pride and Prejudice", + ]); + expect(doc.JSONPath("$..title")).toHaveLength(5); + expect(doc.JSONPath("$.store.books[0,2,-1].title")).toEqual([ + "1984", + "Brave New World", + "Pride and Prejudice", + ]); + expect(doc.JSONPath("$.store.books[0:5:2].title")).toEqual([ + "1984", + "Brave New World", + "Pride and Prejudice", + ]); + expect(doc.JSONPath("$.store.books[?(@.author == $.store.featured_author && @.price < 10)].title")).toEqual(["Animal Farm"]); + expect(doc.JSONPath("$.store.books[?(@.author in $.store.featured_authors)].title")).toEqual(["1984", "Animal Farm", "Pride and Prejudice"]); + expect(doc.JSONPath("$..[?(@.title contains 'Farm')].title")).toEqual([ + "Animal Farm", + ]); + expect(doc.JSONPath("$.store.books[?(count(@.title) == 1)].title")).toHaveLength(5); + expect(doc.JSONPath("$.store.body")).toEqual([body]); + expect(doc.JSONPath("$.outline[0].meta.label")).toEqual(["first"]); + expect(doc.JSONPath("$.outline[-1].meta.label")).toEqual(["last"]); + expect(doc.JSONPath("$")).toEqual([{ store, outline }]); + expect(doc.getByPath("outline/0/0/label")).toBe("child"); + doc.commit(); + let subscriptionHits = 0; + const unsubscribe = doc.subscribeJsonpath("$.store.featured_author", () => { + subscriptionHits += 1; + }); + firstRoot.data.set("unrelated", true); + doc.commit(); + expect(subscriptionHits).toBe(0); + store.set("featured_author", "Jane Austen"); + doc.commit(); + expect(subscriptionHits).toBe(1); + unsubscribe(); + expect(() => doc.subscribeJsonpath("store", () => { })).toThrow(/start with/u); + }); + test("syncs updates in both directions", () => { + const left = new LoroDoc(); + left.setPeerId(1); + left.getList("list").push("A"); + left.getList("list").push("B"); + left.getText("text").insert(0, "hello"); + const right = new LoroDoc(); + right.setPeerId(2); + right.import(left.export({ mode: "update" })); + expect(right.toJSON()).toEqual(left.toJSON()); + const from = right.oplogVersion(); + right.getMap("map").set("x", true); + right.getList("list").delete(0, 1); + left.import(right.export({ mode: "update", from })); + expect(left.toJSON()).toEqual(right.toJSON()); + }); + test("slices update exports at operation boundaries", () => { + const source = new LoroDoc(); + source.setPeerId(1); + source.getText("text").insert(0, "abcde"); + source.commit(); + const prefix = source.export({ + mode: "updates-in-range", + spans: [{ id: { peer: "1", counter: 0 }, len: 2 }], + }); + const version = new VersionVector(new Map([["1", 2]])); + const suffix = source.export({ mode: "update", from: version }); + const target = new LoroDoc(); + target.import(prefix); + expect(target.getText("text").toString()).toBe("ab"); + target.import(suffix); + expect(target.getText("text").toString()).toBe("abcde"); + }); + test("converges when updates are imported in opposite orders", () => { + const base = new LoroDoc(); + base.setPeerId(1); + base.getList("list").push("base"); + base.getText("text").insert(0, "x"); + const initial = base.export({ mode: "update" }); + const left = new LoroDoc(); + left.setPeerId(2); + left.import(initial); + const right = new LoroDoc(); + right.setPeerId(3); + right.import(initial); + const version = left.oplogVersion(); + left.getList("list").insert(0, "left"); + left.getText("text").insert(0, "L"); + left.getMap("map").set("winner", "left"); + const leftUpdate = left.export({ mode: "update", from: version }); + right.getList("list").insert(0, "right"); + right.getText("text").insert(0, "R"); + right.getMap("map").set("winner", "right"); + const rightUpdate = right.export({ mode: "update", from: version }); + left.import(rightUpdate); + right.import(leftUpdate); + expect(left.toJSON()).toEqual(right.toJSON()); + }); + test("keeps concurrent backward insertions in Fugue order", () => { + const left = new LoroDoc(); + left.setPeerId(1); + for (const character of ["o", "l", "l", "e", "H"]) { + left.getText("text").insert(0, character); + } + const right = new LoroDoc(); + right.setPeerId(2); + for (const character of ["!", "d", "l", "r", "o", "W", " "]) { + right.getText("text").insert(0, character); + } + const leftUpdate = left.export({ mode: "update" }); + const rightUpdate = right.export({ mode: "update" }); + left.import(rightUpdate); + right.import(leftUpdate); + expect(left.getText("text").toString()).toBe("Hello World!"); + expect(right.getText("text").toString()).toBe("Hello World!"); + }); + test("does not interleave a concurrent insertion into an existing Fugue run", () => { + const first = new LoroDoc(); + first.setPeerId(1); + const concurrent = new LoroDoc(); + concurrent.setPeerId(2); + const seed = new LoroDoc(); + seed.setPeerId(3); + seed.getText("text").insert(0, "2"); + first.import(seed.export({ mode: "update" })); + first.getText("text").insert(0, "1"); + concurrent.getText("text").insert(0, "b"); + first.import(concurrent.export({ mode: "update" })); + expect(first.getText("text").toString()).toBe("b12"); + }); + test("checks out partial changes and attaches back to the latest state", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "abc"); + doc.commit(); + doc.checkout([{ peer: "1", counter: 1 }]); + expect(text.toString()).toBe("ab"); + expect(doc.isDetached()).toBe(true); + expect(doc.version().toJSON()).toEqual(new Map([["1", 2]])); + expect(doc.oplogVersion().toJSON()).toEqual(new Map([["1", 3]])); + expect(() => text.push("x")).toThrow(/detached document/u); + doc.attach(); + expect(text.toString()).toBe("abc"); + expect(doc.isDetached()).toBe(false); + }); + test("records imports in the oplog while the state is detached", () => { + const source = new LoroDoc(); + source.setPeerId(1); + source.getText("text").insert(0, "base"); + const base = source.export({ mode: "update" }); + const doc = new LoroDoc(); + doc.setPeerId(2); + doc.import(base); + const checkedOut = doc.frontiers(); + doc.checkout(checkedOut); + source.getText("text").push(" latest"); + doc.import(source.export({ mode: "update", from: doc.oplogVersion() })); + expect(doc.getText("text").toString()).toBe("base"); + expect(doc.oplogVersion().compare(doc.version())).toBe(1); + doc.checkoutToLatest(); + expect(doc.getText("text").toString()).toBe("base latest"); + }); + test("holds causally incomplete updates pending until dependencies arrive", () => { + const source = new LoroDoc(); + source.setPeerId(1); + source.getList("list").push("first"); + source.commit(); + const firstVersion = source.oplogVersion(); + const firstUpdate = source.export({ mode: "update" }); + source.getList("list").push("second"); + source.commit(); + const secondUpdate = source.export({ mode: "update", from: firstVersion }); + const target = new LoroDoc(); + const pending = target.import(secondUpdate); + expect(target.getList("list").toArray()).toEqual([]); + expect(pending.success.size).toBe(0); + expect(pending.pending).toEqual(new Map([["1", { start: 1, end: 2 }]])); + const applied = target.import(firstUpdate); + expect(target.getList("list").toArray()).toEqual(["first", "second"]); + expect(applied.success).toEqual(new Map([["1", { start: 0, end: 2 }]])); + expect(applied.pending).toBeNull(); + }); + test("imports the unseen suffix of a change that overlaps local history", () => { + const source = new LoroDoc(); + source.setPeerId(1); + source.getText("text").insert(0, "a"); + source.commit(); + const prefix = source.export({ mode: "update" }); + source.getText("text").insert(1, "b"); + source.commit(); + expect(source.changeCount()).toBe(1); + const target = new LoroDoc(); + target.import(prefix); + const status = target.import(source.export({ mode: "snapshot" })); + expect(target.getText("text").toString()).toBe("ab"); + expect(status.success).toEqual(new Map([["1", { start: 1, end: 2 }]])); + expect(status.pending).toBeNull(); + }); + test("forkAt keeps only history reachable from the requested frontiers", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getList("list").push("first"); + doc.commit(); + const first = doc.frontiers(); + doc.getList("list").push("second"); + doc.commit(); + const fork = doc.forkAt(first); + expect(fork.getList("list").toArray()).toEqual(["first"]); + expect(fork.opCount()).toBe(1); + }); + test("queries version spans, ancestors and changed containers", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + doc.getText("text").insert(0, "Hello"); + doc.commit({ message: "first" }); + const first = doc.frontiers(); + doc.getText("text").push(" World"); + doc.getMap("map").set("ready", true); + doc.commit({ message: "second" }); + const second = doc.frontiers(); + expect(doc.findIdSpansBetween(first, second)).toEqual({ + retreat: [], + forward: [{ peer: "1", counter: 5, length: 7 }], + }); + expect(doc.findIdSpansBetween(second, first)).toEqual({ + retreat: [{ peer: "1", counter: 5, length: 7 }], + forward: [], + }); + expect(new Set(doc.getChangedContainersIn({ peer: "1", counter: 5 }, 7))).toEqual(new Set([doc.getText("text").id, doc.getMap("map").id])); + const messages = []; + doc.travelChangeAncestors(second, (change) => { + messages.push(change.message); + }); + expect(messages).toEqual(["first", "second"]); + }); + test("emits compatible text, list and map diffs", () => { + const doc = new LoroDoc(); + let batch; + doc.subscribe((event) => (batch = event)); + const text = doc.getText("text"); + text.insert(0, "3"); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ type: "text", diff: [{ insert: "3" }] }); + text.insert(1, "12"); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ + type: "text", + diff: [{ retain: 1 }, { insert: "12" }], + }); + const list = doc.getList("list"); + list.insert(0, "3"); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ type: "list", diff: [{ insert: ["3"] }] }); + list.insert(1, "12"); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ + type: "list", + diff: [{ retain: 1 }, { insert: ["12"] }], + }); + const map = doc.getMap("map"); + map.set("a", 1); + map.set("b", 2); + doc.commit(); + expect(batch?.events[0]?.diff).toEqual({ + type: "map", + updated: { a: 1, b: 2 }, + }); + }); + test("emits the final map value after a delete and restore in one commit", () => { + const doc = new LoroDoc(); + doc.setPeerId(2); + const map = doc.getMap("map"); + map.set("title", {}); + doc.commit(); + const batches = []; + doc.subscribe((event) => batches.push(event)); + map.delete("title"); + map.set("title", {}); + doc.attach(); + expect(batches).toHaveLength(1); + expect(batches[0]?.events).toEqual([ + { + target: map.id, + diff: { type: "map", updated: { title: {} } }, + path: ["map"], + }, + ]); + }); + test("preserves visible map transitions but suppresses same-value import winners", () => { + const source = new LoroDoc(); + source.setPeerId(2); + const sourceMap = source.getMap("map"); + sourceMap.set("title", {}); + source.commit(); + const target = new LoroDoc(); + target.setPeerId(1); + target.import(source.export({ mode: "update" })); + const batches = []; + target.subscribe((event) => batches.push(event)); + sourceMap.delete("title"); + sourceMap.set("title", {}); + source.commit(); + target.import(source.export({ mode: "update" })); + expect(batches.map((batch) => batch.events[0]?.diff)).toEqual([ + { type: "map", updated: { title: {} } }, + ]); + const concurrent = new LoroDoc(); + concurrent.setPeerId(3); + concurrent.getMap("map").set("title", {}); + concurrent.commit(); + batches.length = 0; + target.import(concurrent.export({ mode: "update" })); + expect(batches).toEqual([]); + }); + test("does not emit changes for tree metadata deleted in the same commit", () => { + const doc = new LoroDoc(); + const tree = doc.getTree("tree"); + const batches = []; + doc.subscribe((event) => batches.push(event)); + const node = tree.createNode(); + node.data.set("value", []); + tree.delete(node.id); + doc.commit(); + expect(batches).toEqual([]); + }); + test("calculates raw and JSON-compatible diffs without moving document state", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + const child = map.setContainer("child", new LoroMap()); + child.set("value", 1); + doc.getList("list").push("item"); + doc.commit(); + const latest = doc.frontiers(); + const raw = doc.diff([], latest, false); + const rawMap = raw.find(([id]) => id === map.id)?.[1]; + expect(rawMap).toEqual({ type: "map", updated: { child } }); + const json = doc.diff([], latest); + const jsonMap = json.find(([id]) => id === map.id)?.[1]; + expect(jsonMap).toEqual({ + type: "map", + updated: { child: `🦜:${child.id}` }, + }); + expect(doc.toJSON()).toEqual({ map: { child: { value: 1 } }, list: ["item"] }); + }); + test("keeps map tombstones that are visible only in operation history", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + map.set("created-then-deleted", 1); + doc.commit(); + map.delete("created-then-deleted"); + doc.commit(); + expect(doc.diff([], doc.frontiers(), false)).toEqual([ + [map.id, { type: "map", updated: { "created-then-deleted": undefined } }], + ]); + }); + test("applies raw and serialized diffs with fresh child and tree ids", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const map = source.getMap("map"); + const child = map.setContainer("child", new LoroText()); + child.insert(0, "Hello🙂"); + child.mark({ start: 0, end: 5 }, "bold", true); + map.ensureMergeableCounter("revision").increment(3); + const listChild = source.getList("list").pushContainer(new LoroMap()); + listChild.set("nested", true); + const treeNode = source.getTree("tree").createNode(); + treeNode.data.set("name", "root"); + source.commit(); + const raw = source.diff([], source.frontiers(), false); + const serialized = JSON.parse(JSON.stringify(source.diff([], source.frontiers()))); + for (const batch of [raw, serialized]) { + const target = new LoroDoc(); + target.setPeerId(2); + target.applyDiff(batch); + const targetChild = target.getMap("map").get("child"); + expect(targetChild.id).not.toBe(child.id); + expect(targetChild.toDelta()).toEqual(child.toDelta()); + expect(target.toJSON()).toMatchObject({ + map: { child: "Hello🙂", revision: 3 }, + list: [{ nested: true }], + }); + const targetRoot = target.getTree("tree").roots()[0]; + expect(targetRoot?.id).not.toBe(treeNode.id); + expect(targetRoot?.data.toJSON()).toEqual({ name: "root" }); + } + }); + test("reverts by generating operations, including recreated child containers", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const list = doc.getList("list"); + list.push("item1"); + list.push("item2"); + const child = list.pushContainer(new LoroText()); + child.insert(0, "Hello🙂"); + doc.getText("title").insert(0, "A🙂B"); + doc.commit(); + const populated = doc.frontiers(); + child.delete(0, child.length); + list.clear(); + doc.getText("title").update("changed"); + doc.commit(); + doc.revertTo(populated); + expect(doc.toJSON()).toMatchObject({ + list: ["item1", "item2", "Hello🙂"], + title: "A🙂B", + }); + expect(list.get(2).id).not.toBe(child.id); + doc.commit(); + doc.revertTo([]); + expect(doc.toJSON()).toEqual({ list: [], title: "" }); + expect(doc.isDetached()).toBe(false); + }); + test("undoes and redoes local commit groups", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const pushed = []; + const popped = []; + const undo = new UndoManager(doc, { + mergeInterval: 0, + onPush: (isUndo, range) => { + pushed.push([isUndo, range.start, range.end]); + return { value: range.end, cursors: [] }; + }, + onPop: (isUndo) => popped.push(isUndo), + }); + doc.getText("text").insert(0, "hello"); + doc.commit(); + doc.getText("text").insert(5, " world"); + doc.commit(); + expect(undo.canUndo()).toBe(true); + expect(undo.undo()).toBe(true); + expect(doc.toJSON()).toEqual({ text: "hello" }); + expect(undo.undo()).toBe(true); + expect(doc.toJSON()).toEqual({ text: "" }); + expect(undo.canUndo()).toBe(false); + expect(undo.redo()).toBe(true); + expect(doc.toJSON()).toEqual({ text: "hello" }); + expect(undo.redo()).toBe(true); + expect(doc.toJSON()).toEqual({ text: "hello world" }); + expect(undo.canRedo()).toBe(false); + expect(pushed).toHaveLength(6); + expect(popped).toEqual([true, true, false, false]); + }); + test("trims undo history from the front", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const undo = new UndoManager(doc, { mergeInterval: 0, maxUndoSteps: 2 }); + for (const value of ["a", "b", "c"]) { + text.push(value); + doc.commit(); + } + expect(undo.undo()).toBe(true); + expect(undo.undo()).toBe(true); + expect(undo.undo()).toBe(false); + expect(text.toString()).toBe("a"); + }); + test("undo preserves remote and excluded text edits", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const undo = new UndoManager(doc, { + mergeInterval: 0, + excludeOriginPrefixes: ["sys:"], + }); + text.insert(0, "hello"); + doc.commit(); + text.insert(0, "1"); + doc.commit({ origin: "sys:test" }); + text.insert(2, "2"); + doc.commit({ origin: "sys:test" }); + text.insert(4, "3"); + doc.commit({ origin: "sys:test" }); + expect(undo.undo()).toBe(true); + expect(text.toString()).toBe("123"); + expect(undo.redo()).toBe(true); + expect(text.toString()).toBe("1h2e3llo"); + const remote = new LoroDoc(); + remote.setPeerId(2); + remote.import(doc.export({ mode: "snapshot" })); + remote.getText("text").insert(0, "R"); + remote.commit(); + doc.import(remote.export({ mode: "update" })); + expect(undo.undo()).toBe(true); + expect(text.toString()).toBe("R123"); + }); + test("undo follows the local peer after detached checkout", () => { + const doc = new LoroDoc(); + const undo = new UndoManager(doc, { mergeInterval: 0 }); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "Hello"); + doc.commit(); + text.insert(5, " world!"); + doc.commit(); + doc.setDetachedEditing(true); + doc.checkout([{ peer: "1", counter: 4 }]); + expect(undo.canUndo()).toBe(false); + text.insert(5, " alice!"); + doc.commit(); + expect(undo.undo()).toBe(true); + expect(text.toString()).toBe("Hello"); + }); + test("round-trips compressed and uncompressed JSON updates", () => { + const source = new LoroDoc(); + source.setPeerId(19); + source.getText("text").insert(0, "Hello🙂"); + const map = source.getMap("map"); + const child = map.setContainer("child", new LoroMap()); + child.set("value", { nested: [1, true, null] }); + source.getList("list").push("item"); + const movable = source.getMovableList("movable"); + movable.push("first"); + movable.push("second"); + movable.move(1, 0); + movable.set(1, "edited"); + const node = source.getTree("tree").createNode(); + node.data.set("name", "root"); + source.getCounter("counter").increment(2.5); + source.commit(); + const compressed = source.exportJsonUpdates(); + expect(compressed.peers).toEqual(["19"]); + expect(compressed.changes[0]?.id).toBe("0@0"); + expect(source.getOpsInChange({ peer: "19", counter: 0 })).toEqual(source.exportJsonInIdSpan({ + peer: "19", + counter: 0, + length: source.opCount(), + })[0]?.ops); + const uncompressed = source.exportJsonUpdates(undefined, undefined, false); + expect(uncompressed.peers).toBeNull(); + expect(uncompressed.changes[0]?.id).toBe("0@19"); + for (const update of [compressed, JSON.stringify(compressed), uncompressed]) { + const target = new LoroDoc(); + target.importJsonUpdates(update); + expect(target.toJSON()).toEqual(source.toJSON()); + expect(target.export({ mode: "update" })).toBeInstanceOf(Uint8Array); + } + }); + test("redacts JSON update content while preserving child-container structure", () => { + const source = new LoroDoc(); + source.setPeerId(1); + source.getText("text").insert(0, "secret"); + source.getMap("map").set("token", "private"); + const child = source.getMap("map").setContainer("child", new LoroMap()); + child.set("inside", 42); + source.getList("list").push("hidden"); + source.commit(); + const redacted = redactJsonUpdates(source.exportJsonUpdates(), { + "1": [0, source.opCount()], + }); + const target = new LoroDoc(); + target.importJsonUpdates(redacted); + expect(target.toJSON()).toEqual({ + text: "������", + map: { token: null, child: { inside: null } }, + list: [null], + }); + }); + test("exports pending operations as JSON without committing", () => { + const doc = new LoroDoc(); + doc.setPeerId(7); + doc.getText("text").insert(0, "pending"); + const pending = doc.getUncommittedOpsAsJson(); + expect(pending).toMatchObject({ peers: null }); + expect(pending?.changes[0]).toMatchObject({ id: "0@7", ops: [{ counter: 0 }] }); + expect(doc.changeCount()).toBe(0); + const target = new LoroDoc(); + target.importJsonUpdates(pending); + expect(target.toJSON()).toEqual({ text: "pending" }); + }); + test("decodes update, snapshot, and shallow-snapshot blob metadata", () => { + const doc = new LoroDoc(); + doc.setPeerId(5); + doc.setChangeMergeInterval(9); + doc.getText("text").insert(0, "abc"); + doc.commit({ timestamp: 10 }); + const first = doc.version(); + const firstFrontiers = doc.frontiers(); + doc.getMap("map").set("value", 1); + doc.commit({ timestamp: 20 }); + const update = decodeImportBlobMeta(doc.export({ mode: "update", from: first })); + expect(update.mode).toBe("update"); + expect(update.partialStartVersionVector.get("5")).toBe(3); + expect(update.partialEndVersionVector.get("5")).toBe(4); + expect(update.startTimestamp).toBe(20); + expect(update.endTimestamp).toBe(20); + expect(update.changeNum).toBe(1); + const snapshot = decodeImportBlobMeta(doc.export({ mode: "snapshot" })); + expect(snapshot).toMatchObject({ + mode: "snapshot", + startTimestamp: 0, + endTimestamp: 20, + changeNum: 2, + }); + expect(snapshot.partialEndVersionVector.get("5")).toBe(4); + const shallow = decodeImportBlobMeta(doc.export({ mode: "shallow-snapshot", frontiers: firstFrontiers })); + expect(shallow.mode).toBe("shallow-snapshot"); + expect(shallow.startFrontiers).toEqual(firstFrontiers); + expect(shallow.partialStartVersionVector.get("5")).toBe(2); + expect(shallow.partialEndVersionVector.get("5")).toBe(4); + }); + test("merges continuous changes using the wasm change interval", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const updates = []; + source.subscribeLocalUpdates((update) => updates.push(update)); + source.getMap("map").set("first", { shared: 1, left: true }); + source.commit({ timestamp: 110 }); + source.getMap("map").set("second", { right: 2, shared: 3 }); + source.commit({ timestamp: 120 }); + expect(source.changeCount()).toBe(1); + expect(source.getAllChanges().get("1")).toEqual([ + expect.objectContaining({ counter: 0, length: 2, timestamp: 110 }), + ]); + expect(source.exportJsonUpdates().changes).toHaveLength(1); + expect(() => source.debugHistory()).not.toThrow(); + const target = new LoroDoc(); + for (const update of updates) + target.import(update); + expect(target.changeCount()).toBe(1); + expect(target.toJSON()).toEqual(source.toJSON()); + const snapshotTarget = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); + expect(snapshotTarget.toJSON()).toEqual({ + map: { + first: { shared: 1, left: true }, + second: { right: 2, shared: 3 }, + }, + }); + const unmerged = new LoroDoc(); + unmerged.setPeerId(1); + unmerged.setChangeMergeInterval(9); + unmerged.getText("text").insert(0, "a"); + unmerged.commit({ timestamp: 110 }); + unmerged.getText("text").insert(1, "b"); + unmerged.commit({ timestamp: 120 }); + expect(unmerged.getAllChanges().get("1")).toHaveLength(2); + }); + test("appends many merged commits without changing their public history", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const updates = []; + source.subscribeLocalUpdates((update) => updates.push(update)); + const text = source.getText("text"); + for (let index = 0; index < 512; index += 1) { + text.insert(index, "x"); + source.commit(); + } + expect(source.changeCount()).toBe(1); + expect(source.getAllChanges().get("1")).toEqual([ + expect.objectContaining({ counter: 0, length: 512 }), + ]); + const target = new LoroDoc(); + target.importBatch(updates); + expect(target.getText("text").toString()).toBe("x".repeat(512)); + expect(target.changeCount()).toBe(1); + }); + test("coalesces consecutive list inserts in one pending change", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const list = source.getList("list"); + for (let index = 0; index < 128; index += 1) + list.insert(index, index); + expect(source.getUncommittedOpsAsJson()?.changes[0]?.ops).toHaveLength(1); + source.commit(); + expect(source.exportJsonUpdates().changes[0]?.ops).toHaveLength(1); + const target = new LoroDoc(); + target.import(source.export({ mode: "update" })); + expect(target.getList("list").toArray()).toEqual(Array.from({ length: 128 }, (_, index) => index)); + }); + test("preserves next commit options across implicit empty commits only", () => { + const implicit = new LoroDoc(); + implicit.setPeerId(1); + implicit.setNextCommitMessage("kept"); + implicit.export({ mode: "update" }); + implicit.getText("text").insert(0, "a"); + implicit.commit(); + expect(implicit.getAllChanges().get("1")?.[0]?.message).toBe("kept"); + const explicit = new LoroDoc(); + explicit.setPeerId(2); + explicit.setNextCommitMessage("discarded"); + explicit.commit(); + explicit.getText("text").insert(0, "b"); + explicit.commit(); + expect(explicit.getAllChanges().get("2")?.[0]?.message).toBeUndefined(); + }); + test("groups all changes by peer like the wasm Map API", () => { + const first = new LoroDoc(); + first.setPeerId(1); + first.getText("text").insert(0, "a"); + first.commit(); + const second = new LoroDoc(); + second.setPeerId(2); + second.getText("text").insert(0, "b"); + second.commit(); + const target = new LoroDoc(); + target.import(first.export({ mode: "update" })); + target.import(second.export({ mode: "update" })); + expect([...target.getAllChanges().keys()].sort()).toEqual(["1", "2"]); + }); + test("gets the latest peer change at or before a lamport", () => { + const first = new LoroDoc(); + first.setPeerId(1); + const second = new LoroDoc(); + second.setPeerId(2); + first.getText("text").insert(0, "01234"); + second.import(first.export({ mode: "update" })); + second.getText("text").insert(0, "56789"); + first.import(second.export({ mode: "update" })); + first.getText("text").insert(0, "01234"); + first.commit(); + expect(first.getChangeAtLamport("1", 1)).toMatchObject({ + peer: "1", + lamport: 0, + length: 5, + }); + expect(first.getChangeAtLamport("1", 7)?.lamport).toBe(0); + expect(first.getChangeAtLamport("1", 10)?.lamport).toBe(10); + expect(first.getChangeAtLamport("1", 20)?.lamport).toBe(10); + expect(first.getChangeAtLamport("111", 20)).toBeUndefined(); + expect(() => first.getChangeAt({ peer: "1", counter: 99 })).toThrow(/unknown/u); + expect(() => first.getOpsInChange({ peer: "1", counter: 99 })).toThrow(/unknown/u); + }); + test("emits absolute paths and bubbles descendant events to container subscribers", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + const seen = []; + map.subscribe((event) => seen.push(event)); + const subMap = map.setContainer("sub", new LoroMap()); + subMap.set("value", 1); + doc.commit(); + expect(seen.at(-1)?.events.find((event) => event.target === subMap.id)?.path).toEqual(["map", "sub"]); + const list = subMap.setContainer("list", new LoroList()); + list.insert(0, "item"); + const text = list.insertContainer(1, new LoroText()); + doc.commit(); + text.insert(0, "nested"); + doc.commit(); + const textEvent = seen.at(-1)?.events.find((event) => event.target === text.id); + expect(textEvent?.path).toEqual(["map", "sub", "list", 1]); + expect(seen.at(-1)?.currentTarget).toBe(map.id); + expect(doc.getPathToContainer(text.id)).toEqual(["map", "sub", "list", 1]); + text._parentLink = { container: list }; + expect(doc.getPathToContainer(text.id)).toEqual(["map", "sub", "list", 1]); + expect(text._parentLink.binding?.kind).toBe("sequence"); + expect(doc.getByPath("map/sub")).toBe(subMap); + expect(doc.getByPath("map/sub/list/1")).toBe(text); + }); + test("merges first-peer callbacks into the commit and supports ChangeModifier", () => { + const doc = new LoroDoc(); + doc.setPeerId(9); + doc.subscribeFirstCommitFromPeer(({ peer }) => { + doc.getMap("peers").set(peer, `user-${peer}`); + }); + doc.subscribePreCommit(({ changeMeta, modifier }) => { + expect(changeMeta.length).toBe(2); + modifier.setMessage("signed").setTimestamp(123); + }); + doc.getList("list").push("value"); + doc.commit(); + expect(doc.changeCount()).toBe(1); + expect(doc.opCount()).toBe(2); + expect(doc.getMap("peers").get("9")).toBe("user-9"); + expect(doc.getAllChanges().get("9")?.[0]).toMatchObject({ + message: "signed", + timestamp: 123, + }); + }); + test("round trips a TypeScript snapshot", () => { + const source = new LoroDoc(); + source.setPeerId(7); + source.getMap("map").set("nested", { a: [1, true, "x"] }); + source.getText("text").insert(0, "snapshot"); + source.getCounter("counter").increment(3); + source.commit({ message: "snapshot" }); + const target = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); + expect(target.toJSON()).toEqual(source.toJSON()); + expect(target.getAllChanges()).toEqual(source.getAllChanges()); + }); + test("coalesces consecutive text IDs without splitting Unicode scalars", () => { + const source = new LoroDoc(); + source.setPeerId(7); + const text = source.getText("text"); + const value = `${"a".repeat(31)}😀b`; + text.insert(0, `${"a".repeat(31)}😀`); + text.push("b"); + source.commit(); + const snapshot = decodeFastSnapshot(source.export({ mode: "snapshot" })); + const stateStore = decodeStateSnapshotStore(snapshot.state); + expect(stateStore.kind).toBe("sstable"); + if (stateStore.kind !== "sstable") + throw new Error("expected snapshot state"); + const textState = stateStore.containers.find(({ wrapper }) => wrapper.state.kind === ContainerType.Text)?.wrapper.state; + expect(textState?.kind).toBe(ContainerType.Text); + if (textState?.kind !== ContainerType.Text) + throw new Error("expected text state"); + expect(textState.text).toBe(value); + expect(textState.spans.filter(({ length }) => length > 0)).toEqual([ + { peerIndex: 0n, counter: 0, lamportSub: 0, length: 33 }, + ]); + const target = LoroDoc.fromSnapshot(encodeFastSnapshot(snapshot)); + expect(target.getText("text").toString()).toBe(value); + }); + test("bulk-loads multi-span text snapshots and remains editable", () => { + const source = new LoroDoc(); + source.setPeerId(7); + const sourceText = source.getText("text"); + sourceText.insert(0, Array.from({ length: 80 }, (_, index) => `${index % 10}😀`).join("")); + sourceText.delete(6, 6); + sourceText.delete(60, 9); + source.commit(); + const target = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); + const targetText = target.getText("text"); + expect(targetText.toString()).toBe(sourceText.toString()); + targetText.insert(0, "<"); + targetText.insert(31, "中😀"); + targetText.push(">"); + targetText.delete(1, 3); + target.commit(); + const restored = LoroDoc.fromSnapshot(target.export({ mode: "snapshot" })); + expect(restored.getText("text").toString()).toBe(targetText.toString()); + }); + test("rejects duplicate text IDs before changing the document", () => { + const source = new LoroDoc(); + source.setPeerId(8); + source.getText("text").insert(0, "x".repeat(64)); + source.commit(); + const malformed = withDuplicateTextId(decodeFastSnapshot(source.export({ mode: "snapshot" }))); + const target = new LoroDoc(); + const events = []; + target.subscribe((event) => events.push(event)); + expect(() => target.import(malformed)).toThrow(/duplicate sequence IDs/u); + expect(events).toEqual([]); + expect(target.version().toJSON()).toEqual(new Map()); + expect(target.toJSON()).toEqual({}); + }); + test("rejects duplicate text IDs before installing shallow metadata", () => { + const source = new LoroDoc(); + source.setPeerId(9); + source.getText("text").insert(0, "x".repeat(64)); + source.commit(); + const shallowRoot = source.frontiers(); + source.getMap("meta").set("ready", true); + source.commit(); + const malformed = withDuplicateTextId(decodeFastSnapshot(source.export({ mode: "shallow-snapshot", frontiers: shallowRoot }))); + const target = new LoroDoc(); + const events = []; + target.subscribe((event) => events.push(event)); + expect(() => target.import(malformed)).toThrow(/duplicate sequence IDs/u); + expect(events).toEqual([]); + expect(target.isShallow()).toBe(false); + expect(target.version().toJSON()).toEqual(new Map()); + expect(target.toJSON()).toEqual({}); + const concurrent = new LoroDoc(); + concurrent.setPeerId(10); + concurrent.getMap("concurrent").set("value", true); + const batchTarget = new LoroDoc(); + const batchEvents = []; + batchTarget.subscribe((event) => batchEvents.push(event)); + expect(() => batchTarget.importBatch([malformed, concurrent.export({ mode: "update" })])).toThrow(/duplicate sequence IDs/u); + expect(batchEvents).toEqual([]); + expect(batchTarget.isShallow()).toBe(false); + expect(batchTarget.version().toJSON()).toEqual(new Map()); + expect(batchTarget.toJSON()).toEqual({}); + }); + test("keeps snapshot history lazy and owns the deferred bytes", () => { + const source = new LoroDoc(); + source.setPeerId(7); + source.getText("text").insert(0, "snapshot"); + source.getMap("meta").set("ready", true); + source.commit({ message: "seed" }); + const expectedVersion = source.version().toJSON(); + const expectedFrontiers = source.frontiers(); + const expectedChanges = source.getAllChanges(); + const snapshot = source.export({ mode: "snapshot" }); + const target = new LoroDoc(); + const events = []; + target.subscribe((event) => events.push(event)); + const status = target.import(snapshot); + expect(target.toJSON()).toEqual(source.toJSON()); + expect(target.version().toJSON()).toEqual(expectedVersion); + expect(target.frontiers()).toEqual(expectedFrontiers); + expect(target.opCount()).toBe(source.opCount()); + expect(status).toEqual({ + success: new Map([["7", { start: 0, end: 9 }]]), + pending: null, + }); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ by: "import", from: [], to: expectedFrontiers }); + snapshot.fill(0); + expect(target.getAllChanges()).toEqual(expectedChanges); + expect(target.changeCount()).toBe(source.changeCount()); + }); + test("materializes a snapshot before local edits without repeating first-peer events", () => { + const source = new LoroDoc(); + source.setPeerId(7); + source.getText("text").insert(0, "a"); + source.commit(); + const target = new LoroDoc(); + target.setPeerId(7); + target.import(source.export({ mode: "snapshot" })); + const firstPeers = []; + target.subscribeFirstCommitFromPeer(({ peer }) => firstPeers.push(peer)); + target.getText("text").push("b"); + target.commit(); + expect(firstPeers).toEqual([]); + expect(target.getText("text").toString()).toBe("ab"); + expect(target.version().get("7")).toBe(2); + }); + test("materializes a lazy snapshot before importing a later update", () => { + const source = new LoroDoc(); + source.setPeerId(9); + source.getText("text").insert(0, "a"); + source.commit(); + const snapshotVersion = source.version(); + const snapshot = source.export({ mode: "snapshot" }); + source.getText("text").push("b"); + source.commit(); + const update = source.export({ mode: "update", from: snapshotVersion }); + const target = LoroDoc.fromSnapshot(snapshot); + const status = target.import(update); + expect(status).toEqual({ + success: new Map([["9", { start: 1, end: 2 }]]), + pending: null, + }); + expect(target.toJSON()).toEqual({ text: "ab" }); + expect(target.getAllChanges()).toEqual(source.getAllChanges()); + }); + test("keeps deferred history atomic when snapshot metadata is inconsistent", () => { + const source = new LoroDoc(); + source.setPeerId(11); + source.getText("text").insert(0, "seed"); + source.commit(); + const snapshot = decodeFastSnapshot(source.export({ mode: "snapshot" })); + const oplog = decodeSstable(snapshot.oplog).map((entry) => { + if (entry.key.length !== 2 || entry.key[0] !== 0x76 || entry.key[1] !== 0x76) { + return entry; + } + const version = decodePostcardVersionVector(entry.value).map((id) => ({ + ...id, + counter: id.counter + 1, + })); + return { ...entry, value: encodePostcardVersionVector(version) }; + }); + const malformed = encodeFastSnapshot({ + ...snapshot, + oplog: encodeSstable(oplog, { compression: "none" }), + }); + const target = new LoroDoc(); + target.import(malformed); + expect(target.toJSON()).toEqual({ text: "seed" }); + expect(() => target.changeCount()).toThrow(/version does not match/u); + expect(target.toJSON()).toEqual({ text: "seed" }); + expect(() => target.changeCount()).toThrow(/version does not match/u); + }); + test("rejects a corrupt deferred frontier block before changing the document", () => { + const source = new LoroDoc(); + source.setPeerId(12); + source.getText("text").insert(0, "seed"); + source.commit(); + const snapshot = decodeFastSnapshot(source.export({ mode: "snapshot" })); + const oplog = decodeSstable(snapshot.oplog).map((entry) => entry.key.length === 12 ? { ...entry, value: Uint8Array.of(0xff) } : entry); + const malformed = encodeFastSnapshot({ + ...snapshot, + oplog: encodeSstable(oplog, { compression: "none" }), + }); + const target = new LoroDoc(); + const events = []; + target.subscribe((event) => events.push(event)); + expect(() => target.import(malformed)).toThrow(/unexpected end|change block/u); + expect(events).toEqual([]); + expect(target.version().toJSON()).toEqual(new Map()); + expect(target.toJSON()).toEqual({}); + }); + test("owns shallow root bytes until deferred history is materialized", () => { + const source = new LoroDoc(); + source.setPeerId(13); + const original = new Uint8Array(64); + let random = 305419896; + for (let index = 0; index < original.length; index += 1) { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + original[index] = random & 0xff; + } + source.getMap("map").set("bytes", original); + source.commit(); + const shallowRoot = source.frontiers(); + source.getMap("map").set("bytes", Uint8Array.of(9)); + source.commit(); + const bytes = source.export({ + mode: "shallow-snapshot", + frontiers: shallowRoot, + }); + const target = LoroDoc.fromSnapshot(bytes); + expect(target.getMap("map").get("bytes")).toEqual(Uint8Array.of(9)); + bytes.fill(0); + target.checkout(shallowRoot); + expect(target.getMap("map").get("bytes")).toEqual(original); + }); + test("does not install shallow metadata before latest state decoding succeeds", () => { + const source = new LoroDoc(); + source.setPeerId(14); + source.getText("text").insert(0, "a"); + source.commit(); + const shallowRoot = source.frontiers(); + source.getText("text").push("b"); + source.commit(); + const snapshot = decodeFastSnapshot(source.export({ mode: "shallow-snapshot", frontiers: shallowRoot })); + const malformed = encodeFastSnapshot({ + ...snapshot, + state: Uint8Array.of(0xff), + }); + const target = new LoroDoc(); + expect(() => target.import(malformed)).toThrow(/SSTable is too short/u); + expect(target.isShallow()).toBe(false); + expect(target.version().toJSON()).toEqual(new Map()); + expect(target.toJSON()).toEqual({}); + }); + test("exports and imports shallow snapshots at a partial change boundary", () => { + const source = new LoroDoc(); + source.setPeerId(3); + source.getText("text").insert(0, "01234"); + source.commit(); + const shallowRoot = source.frontiers(); + source.getText("text").push("56789"); + source.getMap("meta").set("ready", true); + source.commit(); + const snapshot = source.export({ + mode: "shallow-snapshot", + frontiers: shallowRoot, + }); + const target = LoroDoc.fromSnapshot(snapshot); + expect(target.toJSON()).toEqual(source.toJSON()); + expect(target.isShallow()).toBe(true); + expect(target.shallowSinceFrontiers()).toEqual(shallowRoot); + expect(target.shallowSinceVV().toJSON()).toEqual(new Map([["3", 4]])); + expect(() => target.checkout([])).toThrow(/before the shallow history root/u); + target.checkout(shallowRoot); + expect(target.toJSON()).toEqual({ text: "01234", meta: {} }); + target.checkoutToLatest(); + expect(target.toJSON()).toEqual(source.toJSON()); + const resnapshot = target.export({ mode: "snapshot" }); + const restored = LoroDoc.fromSnapshot(resnapshot); + expect(restored.isShallow()).toBe(true); + expect(restored.toJSON()).toEqual(source.toJSON()); + }); + test("preserves rich-text attributes in snapshots", () => { + const source = new LoroDoc(); + source.setPeerId(7); + const text = source.getText("text"); + text.insert(0, "a😀bc"); + text.mark({ start: 1, end: 3 }, "bold", true); + source.commit(); + const target = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); + expect(target.getText("text").toDelta()).toEqual([ + { insert: "a" }, + { insert: "😀", attributes: { bold: true } }, + { insert: "bc" }, + ]); + }); + test("uses compatible text-style expansion rules", () => { + const doc = new LoroDoc(); + const text = doc.getText("text"); + text.insert(0, "hi"); + text.mark({ start: 0, end: 2 }, "bold", true); + text.insert(2, "!"); + text.insert(0, "("); + expect(text.toDelta()).toEqual([ + { insert: "(" }, + { insert: "hi!", attributes: { bold: true } }, + ]); + const link = doc.getText("link"); + link.insert(0, "x"); + link.mark({ start: 0, end: 1 }, "link", "url"); + link.insert(1, "y"); + expect(link.toDelta()).toEqual([ + { insert: "x", attributes: { link: "url" } }, + { insert: "y" }, + ]); + }); + test("matches sorted map and text edge contracts", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + map.set("z", 1); + map.set("a", 2); + expect(map.entries()).toEqual([ + ["a", 2], + ["z", 1], + ]); + const text = doc.getText("text"); + text.insert(0, "x"); + expect(text.convertPos(0, "invalid", "unicode")).toBeUndefined(); + expect(() => text.mark({ start: 0, end: 1 }, "not-configured", true)).toThrow(/not configured/u); + const detached = new LoroText(); + detached.insert(0, "x"); + expect(() => detached.mark({ start: 0, end: 1 }, "not-configured", true)).not.toThrow(); + }); + test("tracks stable cursors and replaces deleted anchors", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "123"); + const cursor = text.getCursor(0, 0); + const end = text.getCursor(10, 0); + expect(cursor.kind()).toBe("Cursor"); + expect(cursor.containerId()).toBe(text.id); + expect(cursor.pos()).toEqual({ peer: "1", counter: 0 }); + expect(end.side()).toBe(1); + expect(doc.getCursorPos(cursor)).toEqual({ offset: 0, side: 0 }); + text.insert(0, "abc"); + expect(doc.getCursorPos(cursor)).toEqual({ offset: 3, side: 0 }); + const decoded = Cursor.decode(cursor.encode()); + const remote = new LoroDoc(); + remote.import(doc.export({ mode: "update" })); + expect(remote.getCursorPos(decoded)).toEqual({ offset: 3, side: 0 }); + text.delete(3, 1); + doc.commit(); + const result = doc.getCursorPos(cursor); + expect(result.offset).toBe(3); + expect(result.side).toBe(-1); + expect(result.update?.pos()).toEqual({ peer: "1", counter: 1 }); + expect(result.update?.side()).toBe(-1); + }); + test("copies detached child containers and exposes their attached handlers", () => { + const child = new LoroMap(); + child.set("ready", true); + const nestedList = new LoroList(); + nestedList.push(1); + child.setContainer("items", nestedList); + const doc = new LoroDoc(); + doc.setPeerId(3); + const attachedChild = doc.getMap("root").setContainer("child", child); + expect(doc.toJSON()).toEqual({ root: { child: { ready: true, items: [1] } } }); + expect(child.isAttached()).toBe(false); + expect(child.getAttached()).toBe(attachedChild); + expect(nestedList.isAttached()).toBe(false); + expect(nestedList.getAttached()?.isAttached()).toBe(true); + child.set("detached-only", true); + attachedChild.set("attached-only", true); + expect(doc.toJSON()).toEqual({ + root: { child: { ready: true, items: [1], "attached-only": true } }, + }); + }); + test("converges concurrent edits in deterministic mergeable children", () => { + const left = new LoroDoc(); + left.setPeerId(1); + const right = new LoroDoc(); + right.setPeerId(2); + const leftCounter = left.getMap("state").ensureMergeableCounter("revision"); + const rightCounter = right.getMap("state").ensureMergeableCounter("revision"); + leftCounter.increment(1); + rightCounter.increment(1); + const leftUpdate = left.export({ mode: "update" }); + const rightUpdate = right.export({ mode: "update" }); + left.import(rightUpdate); + right.import(leftUpdate); + expect(leftCounter.id).toBe(rightCounter.id); + expect(leftCounter.value).toBe(2); + expect(rightCounter.value).toBe(2); + expect(left.toJSON()).toEqual({ state: { revision: 2 } }); + expect(right.toJSON()).toEqual(left.toJSON()); + }); + test("integrates update batches once in arbitrary blob order", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const text = source.getText("text"); + text.push("a"); + source.commit(); + const firstVersion = source.oplogVersion(); + const first = source.export({ mode: "update" }); + text.push("b"); + source.commit(); + const second = source.export({ mode: "update", from: firstVersion }); + const target = new LoroDoc(); + let events = 0; + target.subscribe(() => { + events += 1; + }); + const status = target.importBatch([second, first]); + expect(target.toJSON()).toEqual({ text: "ab" }); + expect(events).toBe(1); + expect(status.success.get("1")).toEqual({ start: 0, end: 2 }); + expect(status.pending).toBeNull(); + }); + test("integrates mixed snapshot and update batches once", () => { + const source = new LoroDoc(); + source.setPeerId(1); + const text = source.getText("text"); + text.push("a"); + source.commit(); + const firstSnapshot = source.export({ mode: "snapshot" }); + text.push("b"); + source.commit(); + const secondVersion = source.oplogVersion(); + const secondSnapshot = source.export({ mode: "snapshot" }); + text.push("c"); + source.commit(); + const update = source.export({ mode: "update", from: secondVersion }); + const target = new LoroDoc(); + const events = []; + target.subscribe((event) => events.push(event)); + const status = target.importBatch([update, firstSnapshot, secondSnapshot]); + expect(target.toJSON()).toEqual({ text: "abc" }); + expect(events).toHaveLength(1); + expect(events[0].events).toEqual([ + { + target: "cid:root-text:Text", + path: ["text"], + diff: { type: "text", diff: [{ insert: "abc" }] }, + }, + ]); + expect(status.success.get("1")).toEqual({ start: 0, end: 3 }); + expect(status.pending).toBeNull(); + }); + test("keeps mergeable ensure idempotent and guards non-mergeable values", () => { + const doc = new LoroDoc(); + const root = doc.getMap("state"); + const first = root.ensureMergeableMap("profile"); + doc.commit(); + const opCount = doc.opCount(); + expect(root.ensureMergeableMap("profile")).toBe(first); + doc.commit(); + expect(doc.opCount()).toBe(opCount); + root.set("scalar", 1); + expect(() => root.ensureMergeableList("scalar")).toThrow(/non-mergeable value/u); + expect(root.get("scalar")).toBe(1); + root.setContainer("regular", new LoroMap()); + expect(() => root.ensureMergeableMap("regular")).toThrow(/non-mergeable value/u); + }); + test("does not record semantic no-op edits", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + map.set("value", { nested: [1, true, null] }); + const movable = doc.getMovableList("movable"); + movable.push("same"); + doc.commit(); + const opCount = doc.opCount(); + map.set("value", { nested: [1, true, null] }); + map.delete("missing"); + movable.set(0, "same"); + doc.getCounter("counter").increment(0); + doc.commit(); + expect(doc.opCount()).toBe(opCount); + }); + test("does not confuse empty value shapes when detecting no-op edits", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + const movable = doc.getMovableList("movable"); + map.set("value", []); + movable.push([]); + doc.commit(); + const opCount = doc.opCount(); + map.set("value", {}); + movable.set(0, {}); + doc.commit(); + map.set("value", new Uint8Array()); + movable.set(0, new Uint8Array()); + doc.commit(); + expect(map.get("value")).toEqual(new Uint8Array()); + expect(movable.get(0)).toEqual(new Uint8Array()); + expect(doc.opCount()).toBe(opCount + 4); + }); + test("resurfaces preserved state and switches mergeable kinds", () => { + const doc = new LoroDoc(); + const root = doc.getMap("state"); + const text = root.ensureMergeableText("field"); + text.insert(0, "hello"); + doc.commit(); + root.delete("field"); + doc.commit(); + expect(root.get("field")).toBeUndefined(); + expect(text.isDeleted()).toBe(true); + const restored = root.ensureMergeableText("field"); + expect(restored).toBe(text); + expect(restored.toString()).toBe("hello"); + const map = root.ensureMergeableMap("field"); + map.set("value", 1); + expect(doc.toJSON()).toEqual({ state: { field: { value: 1 } } }); + expect(root.ensureMergeableText("field").toString()).toBe("hello"); + }); + test("uses flattened mergeable ids and logical event paths", () => { + const doc = new LoroDoc(); + const root = doc.getMap("state"); + const profile = root.ensureMergeableMap("profile"); + const counter = profile.ensureMergeableCounter("revision"); + const batches = []; + root.subscribe((event) => batches.push(event)); + counter.increment(3); + doc.commit(); + expect(profile.id).toBe("cid:root-🤝:$state>profile:Map"); + expect(counter.id).toBe("cid:root-🤝:$state>profile>revision:Counter"); + expect(doc.getContainerById(counter.id)).toBe(counter); + expect(batches.at(-1)?.events.find((event) => event.target === counter.id)?.path).toEqual(["state", "profile", "revision"]); + }); + test("distinguishes existing roots, normal children, and mergeable children", () => { + const doc = new LoroDoc(); + expect(doc.hasContainer("cid:root-new:Text")).toBe(true); + expect(doc.getContainerById("cid:root-new:Text")?.kind()).toBe("Text"); + expect(doc.hasContainer("cid:0@99:Text")).toBe(false); + expect(() => doc.getText("cid:0@99:Text")).toThrow(/does not exist/); + const root = doc.getMap("state"); + const empty = root.ensureMergeableText("empty"); + expect(doc.hasContainer(empty.id)).toBe(true); + root.delete("empty"); + expect(doc.hasContainer(empty.id)).toBe(false); + const preserved = root.ensureMergeableText("preserved"); + preserved.insert(0, "kept"); + doc.commit(); + root.delete("preserved"); + doc.commit(); + expect(doc.hasContainer(preserved.id)).toBe(true); + expect(preserved.isDeleted()).toBe(true); + }); + test("passes live containers through toJsonWithReplacer", () => { + const doc = new LoroDoc(); + const root = doc.getMap("root"); + root.set("count", 1); + root.set("drop", true); + const body = root.setContainer("body", new LoroText()); + body.insert(0, "hello"); + const seen = []; + expect(doc.toJsonWithReplacer((key, value) => { + if (value instanceof LoroMap || value instanceof LoroText) { + seen.push(`${String(key)}:${value.kind()}`); + } + if (value instanceof LoroText) + return value.toString().toUpperCase(); + if (key === "count") + return 2; + if (key === "drop") + return undefined; + return value; + })).toEqual({ root: { count: 2, body: "HELLO" } }); + expect(seen).toEqual(["root:Map", "body:Text"]); + expect(() => doc.toJsonWithReplacer((_key, value) => (value instanceof LoroText ? root : value))).toThrow(/different container/u); + }); + test("iterates text by contiguous operation runs", () => { + const left = new LoroDoc(); + left.setPeerId(1); + left.getText("text").insert(0, "Hello"); + left.commit(); + const right = left.fork(); + right.setPeerId(2); + right.getText("text").insert(3, " "); + left.import(right.export({ mode: "update", from: left.oplogVersion() })); + const chunks = []; + left.getText("text").iter((chunk) => { + chunks.push(chunk); + }); + expect(chunks).toEqual(["Hel", " ", "lo"]); + const firstChunk = []; + left.getText("text").iter((chunk) => { + firstChunk.push(chunk); + return false; + }); + expect(firstChunk).toEqual(["Hel"]); + expect(left.getText("text").sliceDeltaUtf8(3, 4)).toEqual([{ insert: " " }]); + expect(left.getText("text").getEditorOf(3)).toBe("2"); + }); + test("guards tree cycles and reports move and fractional-index state", () => { + const doc = new LoroDoc(); + doc.setPeerId(7); + const tree = doc.getTree("tree"); + const root = tree.createNode(); + const child = root.createNode(); + const creation = child.creationId(); + expect(child.creator()).toBe("7"); + expect(child.getLastMoveId()).toEqual(creation); + expect(child.fractionalIndex()).toBeTypeOf("string"); + expect(() => root.move(child)).toThrow(/descendant/u); + child.move(undefined, 0); + expect(child.getLastMoveId()).not.toEqual(creation); + tree.disableFractionalIndex(); + expect(tree.isFractionalIndexEnabled()).toBe(false); + expect(child.fractionalIndex()).toBeUndefined(); + tree.enableFractionalIndex(); + expect(tree.isFractionalIndexEnabled()).toBe(true); + expect(() => tree.enableFractionalIndex(256)).toThrow(/unsigned byte/u); + const lastMoveBeforeDelete = child.getLastMoveId(); + tree.delete(child.id); + expect(tree.isNodeDeleted(child.id)).toBe(true); + expect(() => tree.isNodeDeleted("999@999")).toThrow(/does not exist/u); + expect(tree.getNodes({ withDeleted: true })).toHaveLength(2); + expect(root.toJSON()).toMatchObject({ parent: undefined, fractionalIndex: "80" }); + expect(child.parent()?.id).toBe("2147483647@18446744073709551615"); + expect(child.index()).toBeUndefined(); + expect(child.fractionalIndex()).toBeUndefined(); + expect(child.getLastMoveId()).not.toEqual(lastMoveBeforeDelete); + expect(child.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", + }); + }); + test("keeps deleted tree subtrees out of the live forest and snapshots them", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + tree.enableFractionalIndex(0); + const liveRoot = tree.createNode(); + const deletedRoot = tree.createNode(); + const deletedChild = deletedRoot.createNode(); + expect(deletedRoot.toJSON().fractionalIndex).toBe("8180"); + tree.delete(deletedRoot.id); + expect(tree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); + expect(tree.getNodes({ withDeleted: true }).map((node) => node.id)).toEqual([ + liveRoot.id, + deletedRoot.id, + deletedChild.id, + ]); + expect(tree.isNodeDeleted(deletedChild.id)).toBe(true); + expect(deletedChild.isDeleted()).toBe(true); + expect(deletedChild.data.isDeleted()).toBe(true); + expect(deletedRoot.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", + }); + doc.commit(); + const restored = new LoroDoc(); + restored.import(doc.export({ mode: "snapshot" })); + const restoredTree = restored.getTree("tree"); + expect(restoredTree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); + expect(restoredTree.isNodeDeleted(deletedRoot.id)).toBe(true); + expect(restoredTree.isNodeDeleted(deletedChild.id)).toBe(true); + expect(restoredTree.getNodeByID(deletedRoot.id)?.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", + }); + }); + test("round-trips frontiers and reports a package version", () => { + const doc = new LoroDoc(); + doc.setPeerId(11); + doc.getText("text").insert(0, "hello"); + doc.commit(); + const frontiers = doc.frontiers(); + expect(decodeFrontiers(encodeFrontiers(frontiers))).toEqual(frontiers); + expect(LORO_VERSION()).toMatch(/^\d+\.\d+\.\d+(?:[-+].+)?$/u); + }); + test("synchronizes postcard-compatible awareness state", () => { + const a = new AwarenessWasm("1", 30000); + const b = new AwarenessWasm("2", 30000); + a.setLocalState({ cursor: 3, bytes: Uint8Array.of(1, 2, 3) }); + expect(b.apply(a.encode(["1"]))).toEqual({ added: ["1"], updated: [] }); + expect(b.getState("1")).toEqual({ cursor: 3, bytes: Uint8Array.of(1, 2, 3) }); + expect(b.peer()).toBe("2"); + expect(b.length()).toBe(1); + const old = a.encodeAll(); + a.setLocalState({ cursor: 4 }); + const current = a.encodeAll(); + expect(b.apply(current)).toEqual({ added: [], updated: ["1"] }); + expect(b.apply(old)).toEqual({ added: [], updated: [] }); + expect(b.getState("1")).toEqual({ cursor: 4 }); + expect(() => b.apply(Uint8Array.of(0xff))).toThrow(/Failed to decode awareness/u); + const wrapped = new Awareness("3", 30000); + const events = []; + wrapped.addListener((update, origin) => events.push([update, origin])); + wrapped.setLocalState("online"); + expect(wrapped.getLocalState()).toBe("online"); + expect(events).toEqual([[{ updated: [], added: ["3"], removed: [] }, "local"]]); + wrapped.destroy(); + }); + test("synchronizes typed ephemeral state and deletion tombstones", async () => { + const a = new EphemeralStoreWasm(30000); + const b = new EphemeralStoreWasm(30000); + const localUpdates = []; + const events = []; + a.subscribeLocalUpdates((bytes) => localUpdates.push(bytes)); + b.subscribe((event) => events.push(event)); + a.set("cursor", { x: 1, bytes: Uint8Array.of(4, 5) }); + b.apply(localUpdates.at(-1)); + expect(b.getAllStates()).toEqual({ + cursor: { x: 1, bytes: Uint8Array.of(4, 5) }, + }); + expect(events.at(-1)).toEqual({ + by: "import", + added: ["cursor"], + updated: [], + removed: [], + }); + await new Promise((resolve) => setTimeout(resolve, 2)); + a.delete("cursor"); + b.apply(localUpdates.at(-1)); + expect(b.get("cursor")).toBeUndefined(); + expect(events.at(-1)).toEqual({ + by: "import", + added: [], + updated: [], + removed: ["cursor"], + }); + const typed = new EphemeralStore(30000); + typed.set("name", "Ada"); + typed.set("position", 2); + expect(typed.getAllStates()).toEqual({ name: "Ada", position: 2 }); + typed.destroy(); + }); + test("queues nested ephemeral updates until the current callback returns", () => { + const store = new EphemeralStore(30000); + let calls = 0; + store.subscribe(() => { + if (calls === 0) + store.set("a", 2); + calls += 1; + }); + store.set("a", 1); + store.set("b", 2); + store.set("c", 3); + expect(calls).toBe(4); + expect(store.getAllStates()).toEqual({ a: 2, b: 2, c: 3 }); + store.destroy(); + }); + test("provides compatible detached classes and version vectors", () => { + expect(new LoroMap().kind()).toBe("Map"); + expect(new LoroList().kind()).toBe("List"); + expect(new LoroMovableList().kind()).toBe("MovableList"); + expect(new LoroText().kind()).toBe("Text"); + expect(new LoroCounter().kind()).toBe("Counter"); + const version = new VersionVector(new Map([["1", 2]])); + version.setLast({ peer: "2", counter: 3 }); + expect(VersionVector.decode(version.encode()).toJSON()).toEqual(new Map([ + ["1", 2], + ["2", 4], + ])); + }); + test("loads the deep value from a current Rust snapshot", () => { + const doc = LoroDoc.fromSnapshot(fixture("snapshot.blob")); + const expected = JSON.parse(readFileSync(new URL("./fixtures/rust/snapshot.deep.json", import.meta.url), "utf8")); + const actual = doc.toJSON(); + const actualMap = actual.map; + const expectedMap = expected.map; + const actualChildMap = actualMap.child_map; + expect(actualChildMap.a).toBe(1); + expect(actualChildMap.t).toBe("inn"); + expect(actualChildMap.v1476140860).toEqual(Uint8Array.of(0, 1, 2, 46)); + expect(actualMap.child_list).toEqual(expectedMap.child_list); + expect(actualMap.child_mlist).toEqual(expectedMap.child_mlist); + expect(actual.list).toEqual(expected.list); + expect(actual.mlist).toEqual(expected.mlist); + expect(actual.text).toEqual(expected.text); + expect(actual.tree.length).toBe(expected.tree.length); + const version = doc.version().toJSON(); + expect(doc.changeCount()).toBeGreaterThan(0); + expect(doc.version().toJSON()).toEqual(version); + expect(doc.toJSON()).toEqual(actual); + }); +}); diff --git a/loro-js/tests/rust-interop.test.js b/loro-js/tests/rust-interop.test.js new file mode 100644 index 000000000..c14dade47 --- /dev/null +++ b/loro-js/tests/rust-interop.test.js @@ -0,0 +1,176 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, test } from "vitest"; +import { ContainerType, EncodeMode, bytesEqual, decodeChangeBlockKey, decodeChangeBlock, decodeChangeKeys, decodeChangeValueContent, decodeColumnarVecMaybeWrapped, decodeContainerId, decodeContainerArena, decodeDocument, decodeDeleteStartIds, decodeEncodedChangeBlock, decodeEncodedOperations, decodeFastSnapshot, decodeFastUpdates, decodeDeltaRleI32, decodeDeltaRleU32, decodeChangesHeader, decodeChangesMetadata, decodePositionArena, decodePostcardFrontiers, decodePostcardVersionVector, decodeRleU8, decodeRleU32, decodeSstable, decodeStateSnapshotStore, encodeEncodedChangeBlock, encodeChangeBlock, encodeFastSnapshot, encodeFastUpdates, encodeSstable, encodeStateSnapshotStore, } from "../src/codec/index"; +const fixture = (name) => new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); +const textEncoder = new TextEncoder(); +const VV_KEY = textEncoder.encode("vv"); +const FRONTIERS_KEY = textEncoder.encode("fr"); +describe("current Rust interoperability", () => { + test("reads and losslessly reframes a Rust FastUpdates blob", () => { + const bytes = fixture("updates.blob"); + expect(decodeDocument(bytes).mode).toBe(EncodeMode.FastUpdates); + const blocks = decodeFastUpdates(bytes); + expect(blocks.length).toBeGreaterThan(1); + expect(encodeFastUpdates(blocks)).toEqual(bytes); + }); + test("reads every Rust encoded change block and its operation columns", () => { + const blocks = decodeFastUpdates(fixture("updates.blob")); + let operationCount = 0; + for (const bytes of blocks) { + const block = decodeEncodedChangeBlock(bytes); + expect(encodeEncodedChangeBlock(block)).toEqual(bytes); + expect(block.changeCount).toBeGreaterThan(0); + const columns = decodeColumnarVecMaybeWrapped(block.operations); + expect(columns).toHaveLength(4); + const containerIndices = decodeDeltaRleU32(columns[0]); + const props = decodeDeltaRleI32(columns[1]); + const valueTypes = decodeRleU8(columns[2]); + const lengths = decodeRleU32(columns[3]); + expect(props).toHaveLength(containerIndices.length); + expect(valueTypes).toHaveLength(containerIndices.length); + expect(lengths).toHaveLength(containerIndices.length); + operationCount += lengths.length; + } + expect(operationCount).toBeGreaterThan(40); + }); + test("decodes every semantic table in the Rust change blocks", () => { + const blocks = decodeFastUpdates(fixture("updates.blob")); + let changeCount = 0; + let nestedContainerCount = 0; + const valueTypes = new Set(); + for (const bytes of blocks) { + const block = decodeEncodedChangeBlock(bytes); + const header = decodeChangesHeader(block.header, { + changeCount: block.changeCount, + counterStart: block.counterStart, + counterLength: block.counterLength, + lamportStart: block.lamportStart, + lamportLength: block.lamportLength, + }); + const metadata = decodeChangesMetadata(block.changeMetadata, block.changeCount); + const keys = decodeChangeKeys(block.keys); + const containers = decodeContainerArena(block.containerIds, header.peers, keys); + const positions = decodePositionArena(block.positions); + const operations = decodeEncodedOperations(block.operations); + const deletes = decodeDeleteStartIds(block.deleteStartIds); + let remainingValues = block.values; + for (const operation of operations) { + const [value, remaining] = decodeChangeValueContent(operation.valueType, remainingValues); + valueTypes.add(value.type); + remainingValues = remaining; + } + expect(header.lengths).toHaveLength(block.changeCount); + expect(header.dependencies).toHaveLength(block.changeCount); + expect(metadata.timestamps).toHaveLength(block.changeCount); + expect(metadata.commitMessages).toHaveLength(block.changeCount); + expect(operations.reduce((sum, operation) => sum + operation.length, 0)).toBe(block.counterLength); + expect(operations.every((operation) => operation.containerIndex < containers.length)).toBe(true); + expect(deletes.every((entry) => entry.peerIndex < BigInt(header.peers.length))).toBe(true); + expect(remainingValues).toHaveLength(0); + changeCount += block.changeCount; + nestedContainerCount += containers.filter((container) => container.kind === "normal").length; + expect(positions.length).toBeGreaterThanOrEqual(0); + } + expect(changeCount).toBeGreaterThanOrEqual(5); + expect(nestedContainerCount).toBeGreaterThan(0); + expect([...valueTypes]).toEqual(expect.arrayContaining(["loro-value", "string", "raw-tree-move"])); + }); + test("semantically decodes and re-encodes every Rust change block", () => { + const operationTypes = new Set(); + let operationCount = 0; + for (const bytes of decodeFastUpdates(fixture("updates.blob"))) { + const decoded = decodeChangeBlock(bytes); + for (const change of decoded.changes) { + for (const operation of change.operations) { + operationTypes.add(operation.content.type); + operationCount += 1; + } + } + const decodedAgain = decodeChangeBlock(encodeChangeBlock(decoded)); + expect(decodedAgain.changes).toEqual(decoded.changes); + } + expect(operationCount).toBeGreaterThan(40); + expect([...operationTypes]).toEqual(expect.arrayContaining([ + "map-insert", + "text-insert", + "list-insert", + "movable-list-insert", + "tree-create", + ])); + }); + test("reads and losslessly reframes a Rust FastSnapshot blob", () => { + const bytes = fixture("snapshot.blob"); + expect(decodeDocument(bytes).mode).toBe(EncodeMode.FastSnapshot); + const snapshot = decodeFastSnapshot(bytes); + expect(snapshot.oplog.length).toBeGreaterThan(0); + expect(snapshot.state.length).toBeGreaterThan(0); + expect(snapshot.shallowRootState).toHaveLength(0); + expect(encodeFastSnapshot(snapshot)).toEqual(bytes); + }); + test("decodes every Rust ChangeStore and state SSTable block", () => { + const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); + const history = decodeSstable(snapshot.oplog); + const state = decodeSstable(snapshot.state); + const utf8 = new TextDecoder(); + const historyKeys = history.map((entry) => utf8.decode(entry.key)); + expect(historyKeys).toContain("vv"); + expect(historyKeys).toContain("fr"); + expect(history.length).toBeGreaterThan(2); + expect(state.length).toBeGreaterThan(5); + }); + test("decodes semantic keys and versions from a Rust snapshot", () => { + const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); + const history = decodeSstable(snapshot.oplog); + const versionEntry = history.find((entry) => bytesEqual(entry.key, VV_KEY)); + const frontiersEntry = history.find((entry) => bytesEqual(entry.key, FRONTIERS_KEY)); + expect(versionEntry).toBeDefined(); + expect(frontiersEntry).toBeDefined(); + const version = decodePostcardVersionVector(versionEntry.value); + const frontiers = decodePostcardFrontiers(frontiersEntry.value); + expect(version.length).toBeGreaterThan(0); + expect(frontiers.length).toBeGreaterThan(0); + const blockIds = history + .filter((entry) => !bytesEqual(entry.key, VV_KEY) && !bytesEqual(entry.key, FRONTIERS_KEY)) + .map((entry) => decodeChangeBlockKey(entry.key)); + expect(blockIds.length).toBeGreaterThan(0); + expect(blockIds.every((id) => id.counter >= 0)).toBe(true); + const containerIds = decodeSstable(snapshot.state) + .filter((entry) => !bytesEqual(entry.key, FRONTIERS_KEY)) + .map((entry) => decodeContainerId(entry.key)); + const rootNames = containerIds.flatMap((id) => (id.kind === "root" ? [id.name] : [])); + expect(rootNames).toEqual(expect.arrayContaining(["map", "list", "text", "mlist", "tree"])); + expect(containerIds.some((id) => id.kind === "normal")).toBe(true); + }); + test("semantically decodes and re-encodes every Rust container state", () => { + const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); + const store = decodeStateSnapshotStore(snapshot.state); + expect(store.kind).toBe("sstable"); + if (store.kind !== "sstable") { + return; + } + const kinds = new Set(store.containers.map(({ wrapper }) => wrapper.state.kind)); + expect([...kinds]).toEqual(expect.arrayContaining([ + ContainerType.Map, + ContainerType.List, + ContainerType.Text, + ContainerType.Tree, + ContainerType.MovableList, + ])); + expect(store.containers.some(({ id }) => id.kind === "normal")).toBe(true); + const encoded = encodeStateSnapshotStore(store, { compression: "none" }); + expect(decodeStateSnapshotStore(encoded)).toEqual(store); + }); + test("rewrites Rust SSTables into independently decodable TypeScript SSTables", () => { + const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); + for (const table of [snapshot.oplog, snapshot.state]) { + const entries = decodeSstable(table); + const rewritten = encodeSstable(entries, { compression: "none" }); + const decodedAgain = decodeSstable(rewritten); + expect(decodedAgain).toHaveLength(entries.length); + for (let index = 0; index < entries.length; index += 1) { + expect(bytesEqual(decodedAgain[index].key, entries[index].key)).toBe(true); + expect(bytesEqual(decodedAgain[index].value, entries[index].value)).toBe(true); + } + } + }); +}); diff --git a/loro-js/tests/semantic-codecs.test.js b/loro-js/tests/semantic-codecs.test.js new file mode 100644 index 000000000..1d8ec9391 --- /dev/null +++ b/loro-js/tests/semantic-codecs.test.js @@ -0,0 +1,110 @@ +import { describe, expect, test } from "vitest"; +import { ContainerType, bytesToHex, decodeChangeBlockKey, decodeContainerId, decodePostcardContainerId, decodePostcardFrontiers, decodePostcardOptionalContainerId, decodePostcardValue, decodePostcardVersionVector, encodeChangeBlockKey, encodeContainerId, encodePostcardContainerId, encodePostcardFrontiers, encodePostcardOptionalContainerId, encodePostcardValue, encodePostcardVersionVector, } from "../src/codec/index"; +describe("IDs and container IDs", () => { + test("matches the Rust ChangeStore key layout", () => { + const id = { peer: 0x0102030405060708n, counter: -1 }; + const encoded = encodeChangeBlockKey(id); + expect(bytesToHex(encoded)).toBe("0102030405060708ffffffff"); + expect(decodeChangeBlockKey(encoded)).toEqual(id); + }); + test.each([ + [{ kind: "root", name: "root", containerType: ContainerType.Map }, "8004726f6f74"], + [ + { + kind: "normal", + peer: 7n, + counter: 42, + containerType: ContainerType.List, + }, + "0107000000000000002a000000", + ], + ])("matches the raw ContainerID layout", (id, expected) => { + const encoded = encodeContainerId(id); + expect(bytesToHex(encoded)).toBe(expected); + expect(decodeContainerId(encoded)).toEqual(id); + }); + test.each([ + [undefined, "00"], + [ + { kind: "root", name: "root", containerType: ContainerType.Map }, + "010004726f6f7401", + ], + [{ kind: "root", name: "t", containerType: ContainerType.Text }, "0100017400"], + [ + { + kind: "normal", + peer: 7n, + counter: 42, + containerType: ContainerType.List, + }, + "0101075402", + ], + ])("matches Rust postcard Option", (id, expected) => { + const encoded = encodePostcardOptionalContainerId(id); + expect(bytesToHex(encoded)).toBe(expected); + expect(decodePostcardOptionalContainerId(encoded)).toEqual(id); + }); + test("round trips postcard ContainerID without the option wrapper", () => { + const id = { + kind: "normal", + peer: 0xffffffffffffffffn, + counter: -2147483648, + containerType: ContainerType.Tree, + }; + expect(decodePostcardContainerId(encodePostcardContainerId(id))).toEqual(id); + }); +}); +describe("VersionVector and Frontiers", () => { + test("preserves VersionVector order", () => { + const version = [ + { peer: 7n, counter: 42 }, + { peer: 1n, counter: -1 }, + ]; + expect(decodePostcardVersionVector(encodePostcardVersionVector(version))).toEqual(version); + }); + test("canonically sorts Frontiers by peer then counter", () => { + const frontiers = [ + { peer: 7n, counter: 42 }, + { peer: 1n, counter: 0 }, + { peer: 7n, counter: 41 }, + { peer: 1n, counter: -1 }, + ]; + expect(decodePostcardFrontiers(encodePostcardFrontiers(frontiers))).toEqual([ + { peer: 1n, counter: -1 }, + { peer: 1n, counter: 0 }, + { peer: 7n, counter: 41 }, + { peer: 7n, counter: 42 }, + ]); + }); +}); +describe("postcard LoroValue", () => { + const values = [ + { type: "null" }, + { type: "bool", value: false }, + { type: "bool", value: true }, + { type: "double", value: -0 }, + { type: "i64", value: -1n }, + { type: "string", value: "hi 😀" }, + { type: "binary", value: Uint8Array.of(1, 2) }, + { + type: "list", + value: [{ type: "null" }, { type: "i64", value: 1n }], + }, + { + type: "map", + value: [ + ["a", { type: "null" }], + ["b", { type: "i64", value: 2n }], + ], + }, + { + type: "container", + value: { kind: "root", name: "x", containerType: ContainerType.Text }, + }, + ]; + test.each(values)("round trips $type", (value) => { + const decoded = decodePostcardValue(encodePostcardValue(value)); + expect(decoded).toEqual(value); + expect(value.type !== "double" || Object.is(decoded.value, -0)).toBe(true); + }); +}); diff --git a/loro-js/tests/serde-columnar.test.js b/loro-js/tests/serde-columnar.test.js new file mode 100644 index 000000000..dcf522cac --- /dev/null +++ b/loro-js/tests/serde-columnar.test.js @@ -0,0 +1,84 @@ +import { describe, expect, test } from "vitest"; +import { decodeAnyRleU32, decodeBoolRle, decodeColumnarVec, decodeDeltaOfDeltaI64, decodeDeltaRleI32, decodeDeltaRleIsize, decodeDeltaRleU32, decodeDeltaRleUsize, decodeRleU8, encodeBoolRle, encodeColumnarVec, encodeDeltaOfDeltaI64, encodeDeltaRleI32, encodeDeltaRleIsize, encodeDeltaRleU32, encodeDeltaRleUsize, takeAnyRleU32, takeBoolRle, takeDeltaOfDeltaI64, bytesToHex, } from "../src/codec/index"; +describe("serde-columnar", () => { + test("matches the outer vector format", () => { + const encoded = encodeColumnarVec([ + new TextEncoder().encode("abc"), + Uint8Array.of(1, 2), + ]); + expect(bytesToHex(encoded)).toBe("0203616263020102"); + expect(decodeColumnarVec(encoded)).toEqual([ + new TextEncoder().encode("abc"), + Uint8Array.of(1, 2), + ]); + }); + test("decodes known BoolRle and AnyRle runs", () => { + expect(decodeBoolRle(Uint8Array.of(0, 2, 3, 1))).toEqual([ + true, + true, + false, + false, + false, + true, + ]); + expect(decodeRleU8(Uint8Array.of(6, 5, 4, 3))).toEqual([5, 5, 5, 3, 3]); + expect(decodeAnyRleU32(Uint8Array.of(5, 1, 2, 3))).toEqual([1, 2, 3]); + }); + test("supports prefix decoding from concatenated streams", () => { + const boolBytes = Uint8Array.of(0, 2, 3, 1, 99); + const [bools, boolRest] = takeBoolRle(boolBytes, 6); + expect(bools).toEqual([true, true, false, false, false, true]); + expect(boolRest).toEqual(Uint8Array.of(99)); + const anyBytes = Uint8Array.of(5, 1, 2, 3, 99); + const [values, anyRest] = takeAnyRleU32(anyBytes, 3); + expect(values).toEqual([1, 2, 3]); + expect(anyRest).toEqual(Uint8Array.of(99)); + }); + test("round trips BoolRle", () => { + const values = [false, false, true, true, false, true, true, true]; + expect(decodeBoolRle(encodeBoolRle(values))).toEqual(values); + }); + test("decodes the known DeltaRle run encoding", () => { + expect(decodeDeltaRleU32(Uint8Array.of(2, 20, 6, 2, 4, 4))).toEqual([ + 10, 11, 12, 13, 15, 17, + ]); + }); + test("preserves the canonical literal DeltaRle encoding", () => { + expect(bytesToHex(encodeDeltaRleU32([10, 11, 12, 13, 15, 17]))).toBe("0b140202020404"); + expect(bytesToHex(encodeDeltaRleI32([-2, -1, 0, 5, 3]))).toBe("090302020a03"); + }); + test("round trips the full i32 and u32 delta range", () => { + const unsigned = [0, 4294967295, 0, 4294967295]; + const signed = [-2147483648, 2147483647, -2147483648]; + expect(decodeDeltaRleU32(encodeDeltaRleU32(unsigned))).toEqual(unsigned); + expect(decodeDeltaRleI32(encodeDeltaRleI32(signed))).toEqual(signed); + }); + test("round trips DeltaRle integer variants", () => { + const unsigned = [0, 1, 2, 10, 11, 5]; + const signed = [-2, -1, 0, 5, 3]; + const wideUnsigned = unsigned.map(BigInt); + const wideSigned = signed.map(BigInt); + expect(decodeDeltaRleU32(encodeDeltaRleU32(unsigned))).toEqual(unsigned); + expect(decodeDeltaRleI32(encodeDeltaRleI32(signed))).toEqual(signed); + expect(decodeDeltaRleUsize(encodeDeltaRleUsize(wideUnsigned))).toEqual(wideUnsigned); + expect(decodeDeltaRleIsize(encodeDeltaRleIsize(wideSigned))).toEqual(wideSigned); + }); + test.each([ + { values: [], hex: "0000" }, + { values: [5n], hex: "010a00" }, + { values: [1n, 2n, 3n], hex: "010202a000" }, + ])("matches known DeltaOfDelta bytes $hex", ({ values, hex }) => { + const encoded = encodeDeltaOfDeltaI64(values); + expect(bytesToHex(encoded)).toBe(hex); + expect(decodeDeltaOfDeltaI64(encoded)).toEqual(values); + }); + test("takes a DeltaOfDelta prefix without consuming the next stream", () => { + const encoded = encodeDeltaOfDeltaI64([10n, 12n, 15n]); + const combined = new Uint8Array(encoded.length + 2); + combined.set(encoded); + combined.set([7, 8], encoded.length); + const [values, rest] = takeDeltaOfDeltaI64(combined, 3); + expect(values).toEqual([10n, 12n, 15n]); + expect(rest).toEqual(Uint8Array.of(7, 8)); + }); +}); diff --git a/loro-js/tests/sstable.test.js b/loro-js/tests/sstable.test.js new file mode 100644 index 000000000..7559679e8 --- /dev/null +++ b/loro-js/tests/sstable.test.js @@ -0,0 +1,33 @@ +import { describe, expect, test } from "vitest"; +import { LoroDecodeError, decodeSstable, encodeSstable } from "../src/codec/index"; +describe("SSTable", () => { + test("uses zero bytes for an empty KV store", () => { + expect(encodeSstable([])).toEqual(new Uint8Array()); + expect(decodeSstable(new Uint8Array())).toEqual([]); + }); + test.each(["none", "auto", "lz4"])("round trips with %s compression", (compression) => { + const entries = [ + { key: Uint8Array.of(1), value: new Uint8Array() }, + { key: Uint8Array.of(1, 2), value: Uint8Array.of(4, 5) }, + { key: Uint8Array.of(2), value: new Uint8Array(5000).fill(9) }, + ]; + expect(decodeSstable(encodeSstable(entries, { compression }))).toEqual(entries); + }); + test("sorts keys and rejects duplicates", () => { + const encoded = encodeSstable([ + { key: Uint8Array.of(2), value: Uint8Array.of(2) }, + { key: Uint8Array.of(1), value: Uint8Array.of(1) }, + ]); + expect(decodeSstable(encoded).map((entry) => entry.key[0])).toEqual([1, 2]); + expect(() => encodeSstable([ + { key: Uint8Array.of(1), value: new Uint8Array() }, + { key: Uint8Array.of(1), value: new Uint8Array() }, + ])).toThrow("unique"); + }); + test("checks the document-level block checksum", () => { + const encoded = encodeSstable([{ key: Uint8Array.of(1), value: Uint8Array.of(2) }]); + const corrupted = encoded.slice(); + corrupted[6] = corrupted[6] ^ 1; + expect(() => decodeSstable(corrupted)).toThrow(LoroDecodeError); + }); +}); diff --git a/loro-js/tests/state-snapshot.test.js b/loro-js/tests/state-snapshot.test.js new file mode 100644 index 000000000..3441f1752 --- /dev/null +++ b/loro-js/tests/state-snapshot.test.js @@ -0,0 +1,141 @@ +import { describe, expect, test } from "vitest"; +import { ContainerType, decodeContainerStateSnapshot, decodeContainerStateWrapper, decodeStateSnapshotStore, encodeContainerStateSnapshot, encodeContainerStateWrapper, encodePositionArena, encodeStateSnapshotStore, unknownContainerType, } from "../src/codec/index"; +const peer = 0x0102030405060708n; +describe("container state snapshot codec", () => { + test("canonicalizes and round trips map state", () => { + const state = { + kind: ContainerType.Map, + values: [ + ["b", { type: "bool", value: true }], + ["a", { type: "i64", value: 1n }], + ], + deletedKeys: ["d", "c"], + peers: [peer], + metadata: [ + { key: "d", peerIndex: 0n, lamport: 4n }, + { key: "b", peerIndex: 0n, lamport: 2n }, + { key: "a", peerIndex: 0n, lamport: 1n }, + { key: "c", peerIndex: 0n, lamport: 3n }, + ], + }; + expect(decodeContainerStateSnapshot(ContainerType.Map, encodeContainerStateSnapshot(state))).toEqual({ + ...state, + values: [...state.values].reverse(), + deletedKeys: [...state.deletedKeys].reverse(), + metadata: [...state.metadata].sort((left, right) => left.key.localeCompare(right.key)), + }); + }); + test("round trips list, text, tree, movable-list, counter and unknown state", () => { + const states = [ + { + kind: ContainerType.List, + values: [{ type: "string", value: "a" }, { type: "null" }], + peers: [peer], + ids: [ + { peerIndex: 0n, counter: 0, lamportSub: 0 }, + { peerIndex: 0n, counter: 1, lamportSub: 1 }, + ], + }, + { + kind: ContainerType.Text, + text: "a😀", + peers: [peer], + spans: [ + { peerIndex: 0n, counter: 0, lamportSub: 0, length: 2 }, + { peerIndex: 0n, counter: 2, lamportSub: 0, length: 0 }, + ], + keys: ["bold"], + marks: [{ keyIndex: 0, value: { type: "bool", value: true }, info: 1 }], + }, + { + kind: ContainerType.Tree, + peers: [peer], + nodes: [ + { + peerIndex: 0n, + counter: 3, + parentIndexPlusTwo: 0n, + lastSetPeerIndex: 0n, + lastSetCounter: 3, + lastSetLamportSub: 0, + fractionalIndexIndex: 0, + }, + ], + positions: [Uint8Array.of(1, 2, 3)], + reserved: new Uint8Array(), + }, + { + kind: ContainerType.MovableList, + values: [{ type: "i64", value: 1n }], + peers: [peer], + items: [ + { + invisibleListItems: 0n, + positionIdEqualsElementId: true, + elementIdEqualsLastSetId: true, + }, + { + invisibleListItems: 0n, + positionIdEqualsElementId: false, + elementIdEqualsLastSetId: false, + }, + ], + listItemIds: [{ peerIndex: 0n, counter: 4, lamportSub: 0 }], + elementIds: [{ peerIndex: 0n, lamport: 4 }], + lastSetIds: [{ peerIndex: 0n, lamport: 5 }], + }, + { kind: ContainerType.Counter, bits: 0x3ff0000000000000n }, + { kind: unknownContainerType(19), payload: Uint8Array.of(1, 2, 3) }, + ]; + for (const state of states) { + expect(decodeContainerStateSnapshot(state.kind, encodeContainerStateSnapshot(state))).toEqual(state); + } + }); + test("round trips empty tree position arena in its non-empty Rust encoding", () => { + const state = { + kind: ContainerType.Tree, + peers: [], + nodes: [], + positions: [], + reserved: new Uint8Array(), + }; + expect(encodePositionArena([], { encodeEmpty: true }).length).toBeGreaterThan(0); + expect(decodeContainerStateSnapshot(ContainerType.Tree, encodeContainerStateSnapshot(state))).toEqual(state); + }); +}); +describe("state wrapper and store codec", () => { + const state = { + kind: ContainerType.Map, + values: [["answer", { type: "i64", value: 42n }]], + deletedKeys: [], + peers: [peer], + metadata: [{ key: "answer", peerIndex: 0n, lamport: 1n }], + }; + const wrapper = { + containerType: ContainerType.Map, + depth: 1n, + parent: undefined, + state, + }; + test("round trips a wrapper", () => { + expect(decodeContainerStateWrapper(encodeContainerStateWrapper(wrapper))).toEqual(wrapper); + }); + test("preserves absent and empty stores", () => { + for (const store of [{ kind: "absent" }, { kind: "empty" }]) { + expect(decodeStateSnapshotStore(encodeStateSnapshotStore(store))).toEqual(store); + } + }); + test("round trips an SSTable store", () => { + const store = { + kind: "sstable", + frontiers: [{ peer, counter: 0 }], + containers: [ + { + id: { kind: "root", name: "map", containerType: ContainerType.Map }, + wrapper, + }, + ], + }; + expect(decodeStateSnapshotStore(encodeStateSnapshotStore(store, { compression: "none" }))).toEqual(store); + }); +}); diff --git a/loro-js/tests/version-transition.test.js b/loro-js/tests/version-transition.test.js new file mode 100644 index 000000000..4517c6982 --- /dev/null +++ b/loro-js/tests/version-transition.test.js @@ -0,0 +1,790 @@ +import { describe, expect, test, vi } from "vitest"; +import { LoroDoc } from "../src/runtime/document"; +describe("indexed version transitions", () => { + test("reports tree checkout deletions at their original sibling indices", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const emptyFrontiers = doc.frontiers(); + const tree = doc.getTree("tree"); + const first = tree.createNode(); + const second = tree.createNode(); + doc.commit(); + let batch; + doc.subscribe((event) => { + batch = event; + }); + doc.checkout(emptyFrontiers); + expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ + type: "tree", + diff: [ + { + target: first.id, + action: "delete", + oldParent: undefined, + oldIndex: 0, + }, + { + target: second.id, + action: "delete", + oldParent: undefined, + oldIndex: 1, + }, + ], + }); + }); + test("revives tree metadata events when checkout restores a node", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const node = tree.createNode(); + node.data.set("value", {}); + doc.commit(); + const liveFrontiers = doc.frontiers(); + tree.delete(node.id); + doc.commit(); + let batch; + doc.subscribe((event) => { + batch = event; + }); + doc.checkout(liveFrontiers); + expect(batch?.events.find(({ target }) => target === node.data.id)?.diff).toEqual({ + type: "map", + updated: { value: {} }, + }); + expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ + type: "tree", + diff: [ + { + target: node.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: node.fractionalIndex(), + }, + ], + }); + }); + test("retreats and restores indexed container state", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const list = doc.getList("list"); + const map = doc.getMap("map"); + const tree = doc.getTree("tree"); + const counter = doc.getCounter("counter"); + text.insert(0, "abcdef"); + list.insert(0, "a"); + list.insert(1, "b"); + map.set("value", 1); + map.set("removed", true); + const firstRoot = tree.createNode(); + const deletedRoot = tree.createNode(); + counter.increment(2); + doc.commit(); + const baseFrontiers = doc.frontiers(); + const base = doc.toJSON(); + text.delete(1, 2); + text.insert(2, "XY"); + list.delete(0, 1); + list.insert(1, "c"); + map.set("value", 2); + map.delete("removed"); + const child = tree.createNode(firstRoot.id); + tree.delete(deletedRoot.id); + counter.increment(3); + doc.commit(); + const latest = doc.toJSON(); + for (let iteration = 0; iteration < 3; iteration += 1) { + doc.checkout(baseFrontiers); + expect(doc.toJSON()).toEqual(base); + expect(tree.has(child.id)).toBe(false); + expect(tree.has(deletedRoot.id)).toBe(true); + doc.checkoutToLatest(); + expect(doc.toJSON()).toEqual(latest); + expect(tree.has(child.id)).toBe(true); + expect(tree.isNodeDeleted(deletedRoot.id)).toBe(true); + } + }); + test("switches between concurrent indexed branches", () => { + const seed = new LoroDoc(); + seed.setPeerId(1); + seed.getText("text").insert(0, "base"); + seed.getMap("map").set("base", true); + seed.commit(); + const snapshot = seed.export({ mode: "snapshot" }); + const left = LoroDoc.fromSnapshot(snapshot); + left.setPeerId(2); + left.getText("text").insert(0, "L"); + left.getMap("map").set("left", true); + left.getCounter("counter").increment(2); + left.getTree("tree").createNode(); + left.commit(); + const leftFrontiers = left.frontiers(); + const leftValue = left.toJSON(); + const right = LoroDoc.fromSnapshot(snapshot); + right.setPeerId(3); + right.getText("text").push("R"); + right.getMap("map").set("right", true); + right.getCounter("counter").increment(3); + right.getTree("tree").createNode(); + right.commit(); + const rightFrontiers = right.frontiers(); + const rightValue = right.toJSON(); + left.import(right.export({ mode: "update", from: left.oplogVersion() })); + const merged = left.toJSON(); + left.checkout(leftFrontiers); + expect(left.toJSON()).toEqual(leftValue); + left.checkout(rightFrontiers); + expect(left.toJSON()).toEqual(rightValue); + left.checkoutToLatest(); + expect(left.toJSON()).toEqual(merged); + }); + test("matches a rebuilt fork across conflicting concurrent versions", () => { + const seed = new LoroDoc(); + seed.setPeerId(1); + const text = seed.getText("text"); + text.insert(0, "abc"); + seed.getList("list").push("base"); + seed.getMap("map").set("value", 0); + const firstRoot = seed.getTree("tree").createNode(); + const secondRoot = seed.getTree("tree").createNode(); + seed.getCounter("counter").increment(1); + seed.commit(); + const baseFrontiers = seed.frontiers(); + const left = seed.fork(); + left.setPeerId(2); + left.getText("text").delete(1, 1); + left.getText("text").insert(1, "L"); + left.getList("list").push("left"); + left.getMap("map").set("value", 1); + left.getTree("tree").move(secondRoot.id, firstRoot.id); + left.getCounter("counter").increment(2); + left.commit(); + const leftFrontiers = left.frontiers(); + const right = seed.fork(); + right.setPeerId(3); + right.getText("text").delete(1, 1); + right.getText("text").insert(2, "R"); + right.getList("list").insert(0, "right"); + right.getMap("map").set("value", 2); + right.getTree("tree").delete(secondRoot.id); + right.getCounter("counter").increment(3); + right.commit(); + const rightFrontiers = right.frontiers(); + left.import(right.export({ mode: "update", from: seed.oplogVersion() })); + const mergedFrontiers = left.frontiers(); + const resets = [ + left.getText("text"), + left.getList("list"), + left.getMap("map"), + left.getTree("tree"), + left.getCounter("counter"), + ].map((container) => vi.spyOn(container, "_reset")); + for (const frontiers of [ + leftFrontiers, + baseFrontiers, + rightFrontiers, + mergedFrontiers, + baseFrontiers, + mergedFrontiers, + ]) { + const expected = left.forkAt(frontiers); + left.checkout(frontiers); + expect(left.toJSON()).toEqual(expected.toJSON()); + expect(left.getText("text").toDelta()).toEqual(expected.getText("text").toDelta()); + } + expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); + }); + test("emits a compact replayable diff without resetting indexed containers", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const list = doc.getList("list"); + const map = doc.getMap("map"); + const tree = doc.getTree("tree"); + const counter = doc.getCounter("counter"); + text.insert(0, "a".repeat(16384)); + list.push("a"); + list.push("b"); + map.set("value", 1); + const root = tree.createNode(); + counter.increment(1); + doc.commit(); + const baseFrontiers = doc.frontiers(); + text.delete(8192, 1); + text.insert(8192, "X"); + list.delete(0, 1); + list.push("c"); + map.set("value", 2); + tree.createNode(root.id); + counter.increment(2); + doc.commit(); + const latestMirror = doc.fork(); + let batch; + doc.subscribe((event) => { + batch = event; + }); + const resets = [text, list, map, tree, counter].map((container) => vi.spyOn(container, "_reset")); + doc.checkout(baseFrontiers); + expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); + expect(batch).toBeDefined(); + latestMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(latestMirror.toJSON()).toEqual(doc.toJSON()); + const textDiff = batch.events.find(({ target }) => target === text.id)?.diff; + expect(JSON.stringify(textDiff).length).toBeLessThan(256); + const baseMirror = doc.forkAt(baseFrontiers); + batch = undefined; + doc.checkoutToLatest(); + expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); + expect(batch).toBeDefined(); + baseMirror.applyDiff(batch.events + .filter(({ target }) => target !== tree.id) + .map(({ target, diff }) => [target, diff])); + expect(baseMirror.getText("text").toDelta()).toEqual(text.toDelta()); + expect(baseMirror.getList("list").toJSON()).toEqual(list.toJSON()); + expect(baseMirror.getMap("map").toJSON()).toEqual(map.toJSON()); + expect(baseMirror.getCounter("counter").value).toBe(counter.value); + expect(batch.events.find(({ target }) => target === tree.id)?.diff.type).toBe("tree"); + }); + test("calculates a retreat diff incrementally and restores the current state", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const list = doc.getList("list"); + const map = doc.getMap("map"); + const counter = doc.getCounter("counter"); + text.insert(0, "a".repeat(16384)); + list.push("base"); + map.set("value", 1); + counter.increment(1); + doc.commit(); + const baseFrontiers = doc.frontiers(); + text.delete(8192, 1); + text.insert(8192, "X"); + list.push("latest"); + map.set("value", 2); + counter.increment(2); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const latest = doc.toJSON(); + const mirror = doc.fork(); + const expectedBase = doc.forkAt(baseFrontiers); + const resets = [text, list, map, counter].map((container) => vi.spyOn(container, "_reset")); + const reverse = doc.diff(latestFrontiers, baseFrontiers, false); + expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); + expect(doc.toJSON()).toEqual(latest); + expect(JSON.stringify(reverse).length).toBeLessThan(512); + mirror.applyDiff(reverse); + expect(mirror.getText("text").toDelta()).toEqual(expectedBase.getText("text").toDelta()); + expect(mirror.getList("list").toJSON()).toEqual(expectedBase.getList("list").toJSON()); + expect(mirror.getMap("map").toJSON()).toEqual(expectedBase.getMap("map").toJSON()); + expect(mirror.getCounter("counter").value).toBe(expectedBase.getCounter("counter").value); + }); + test("retreats and reapplies a full-range sequence deletion", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "x".repeat(16384)); + doc.commit(); + const beforeDelete = doc.frontiers(); + const visibleMirror = doc.fork(); + text.delete(0, text.length); + doc.commit(); + const afterDelete = doc.frontiers(); + const deletedMirror = doc.fork(); + doc.checkout(beforeDelete); + expect(text.toString()).toBe("x".repeat(16384)); + doc.checkout(afterDelete); + expect(text.toString()).toBe(""); + let batch; + doc.subscribe((event) => { + batch = event; + }); + doc.checkout(beforeDelete); + expect(batch).toBeDefined(); + deletedMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(deletedMirror.getText("text").toString()).toBe(text.toString()); + batch = undefined; + doc.checkout(afterDelete); + expect(batch).toBeDefined(); + visibleMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(visibleMirror.getText("text").toString()).toBe(text.toString()); + }); + test("retreats and restores a full-range sequence insertion", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const beforeInsert = doc.frontiers(); + const emptyMirror = doc.fork(); + text.insert(0, "x".repeat(16384)); + doc.commit(); + const afterInsert = doc.frontiers(); + const visibleMirror = doc.fork(); + doc.checkout(beforeInsert); + expect(text.toString()).toBe(""); + doc.checkout(afterInsert); + expect(text.toString()).toBe("x".repeat(16384)); + let batch; + doc.subscribe((event) => { + batch = event; + }); + doc.checkout(beforeInsert); + expect(batch).toBeDefined(); + visibleMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(visibleMirror.getText("text").toString()).toBe(text.toString()); + batch = undefined; + doc.checkout(afterInsert); + expect(batch).toBeDefined(); + emptyMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(emptyMirror.getText("text").toString()).toBe(text.toString()); + }); + test("keeps a range hidden while another concurrent deletion is included", () => { + const seed = new LoroDoc(); + seed.setPeerId(1); + seed.getText("text").insert(0, "x".repeat(4096)); + seed.commit(); + const baseFrontiers = seed.frontiers(); + const left = seed.fork(); + left.setPeerId(2); + left.getText("text").delete(0, 4096); + left.commit(); + const leftFrontiers = left.frontiers(); + const right = seed.fork(); + right.setPeerId(3); + right.getText("text").delete(0, 4096); + right.commit(); + const rightFrontiers = right.frontiers(); + left.import(right.export({ mode: "update", from: seed.oplogVersion() })); + expect(left.getText("text").toString()).toBe(""); + left.checkout(rightFrontiers); + expect(left.getText("text").toString()).toBe(""); + left.checkout(leftFrontiers); + expect(left.getText("text").toString()).toBe(""); + left.checkout(baseFrontiers); + expect(left.getText("text").toString()).toBe("x".repeat(4096)); + left.checkoutToLatest(); + expect(left.getText("text").toString()).toBe(""); + }); + test("retreats rich-text marks through per-element style history", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "abcdef"); + text.mark({ start: 1, end: 5 }, "bold", true); + doc.commit(); + const baseFrontiers = doc.frontiers(); + text.unmark({ start: 2, end: 4 }, "bold"); + text.insert(3, "X"); + text.mark({ start: 0, end: 3 }, "bold", true); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const reset = vi.spyOn(text, "_reset"); + for (const frontiers of [ + baseFrontiers, + latestFrontiers, + baseFrontiers, + latestFrontiers, + ]) { + const expected = doc.forkAt(frontiers).getText("text").toDelta(); + doc.checkout(frontiers); + expect(text.toDelta()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("emits replayable range-based rich-text mark transitions", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "a".repeat(16384)); + doc.commit(); + const baseFrontiers = doc.frontiers(); + const baseMirror = doc.fork(); + text.mark({ start: 1024, end: 15360 }, "bold", true); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const latestMirror = doc.fork(); + let batch; + doc.subscribe((event) => { + batch = event; + }); + doc.checkout(baseFrontiers); + expect(batch).toBeDefined(); + latestMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(latestMirror.getText("text").toDelta()).toEqual(text.toDelta()); + expect(JSON.stringify(batch.events[0].diff).length).toBeLessThan(256); + batch = undefined; + doc.checkout(latestFrontiers); + expect(batch).toBeDefined(); + baseMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(baseMirror.getText("text").toDelta()).toEqual(text.toDelta()); + expect(JSON.stringify(batch.events[0].diff).length).toBeLessThan(256); + }); + test("keeps a subscribed full-range delete and mark transition compact", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + text.insert(0, "x".repeat(16384)); + doc.commit(); + const baseFrontiers = doc.frontiers(); + const mirror = doc.fork(); + text.mark({ start: 0, end: text.length }, "bold", true); + text.delete(0, text.length); + doc.commit(); + const latestFrontiers = doc.frontiers(); + doc.checkout(baseFrontiers); + let batch; + doc.subscribe((event) => { + batch = event; + }); + doc.checkout(latestFrontiers); + expect(batch).toBeDefined(); + expect(JSON.stringify(batch.events[0].diff).length).toBeLessThan(256); + mirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(mirror.getText("text").toDelta()).toEqual(text.toDelta()); + }); + test("retreats movable-list value writes without rebuilding its order", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const list = doc.getMovableList("list"); + list.push("a"); + list.push("b"); + list.push("c"); + doc.commit(); + const baseFrontiers = doc.frontiers(); + list.set(1, "B"); + doc.commit(); + const middleFrontiers = doc.frontiers(); + list.set(1, "latest"); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + baseFrontiers, + latestFrontiers, + middleFrontiers, + baseFrontiers, + latestFrontiers, + ]) { + const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); + doc.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("retreats a movable-list transaction that mixes move, delete, and insert", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const list = doc.getMovableList("list"); + for (const value of ["a", "b", "c", "d"]) + list.push(value); + doc.commit(); + const baseFrontiers = doc.frontiers(); + list.move(1, 3); + list.delete(3, 1); + list.insert(1, "x"); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + baseFrontiers, + latestFrontiers, + baseFrontiers, + latestFrontiers, + ]) { + const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); + doc.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("switches concurrent movable-list moves by replaying branch positions", () => { + const seed = new LoroDoc(); + seed.setPeerId(1); + const seedList = seed.getMovableList("list"); + for (const value of ["a", "b", "c", "d"]) + seedList.push(value); + seed.commit(); + const left = seed.fork(); + left.setPeerId(2); + left.getMovableList("list").move(0, 2); + left.commit(); + const leftFrontiers = left.frontiers(); + const leftValue = left.getMovableList("list").toJSON(); + const right = seed.fork(); + right.setPeerId(3); + right.getMovableList("list").move(0, 1); + right.commit(); + const rightFrontiers = right.frontiers(); + const rightValue = right.getMovableList("list").toJSON(); + left.import(right.export({ mode: "update", from: seed.oplogVersion() })); + const mergedFrontiers = left.frontiers(); + const mergedValue = left.getMovableList("list").toJSON(); + const list = left.getMovableList("list"); + const reset = vi.spyOn(list, "_reset"); + for (const [frontiers, expected] of [ + [leftFrontiers, leftValue], + [rightFrontiers, rightValue], + [leftFrontiers, leftValue], + [mergedFrontiers, mergedValue], + ]) { + left.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + const diffResetCalls = reset.mock.calls.length; + const diff = left.diff(leftFrontiers, rightFrontiers, false); + const mirror = left.forkAt(leftFrontiers); + mirror.applyDiff(diff); + expect(mirror.getMovableList("list").toJSON()).toEqual(rightValue); + expect(reset).toHaveBeenCalledTimes(diffResetCalls); + }); + test("retreats and reapplies a sequential movable-list move suffix", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const list = doc.getMovableList("list"); + for (const value of ["a", "b", "c", "d"]) + list.push(value); + doc.commit(); + const baseFrontiers = doc.frontiers(); + list.move(0, 3); + doc.commit(); + const middleFrontiers = doc.frontiers(); + list.move(1, 0); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + middleFrontiers, + baseFrontiers, + latestFrontiers, + middleFrontiers, + latestFrontiers, + ]) { + const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); + doc.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("retreats a movable-list move suffix through recorded anchors", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const list = doc.getMovableList("list"); + for (const value of ["a", "b", "c", "d"]) + list.push(value); + doc.commit(); + const baseFrontiers = doc.frontiers(); + list.move(0, 2); + doc.commit(); + const middleFrontiers = doc.frontiers(); + list.move(3, 1); + doc.commit(); + const latestFrontiers = doc.frontiers(); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + baseFrontiers, + middleFrontiers, + latestFrontiers, + baseFrontiers, + latestFrontiers, + ]) { + const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); + doc.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("switches directly between concurrent movable-list move branches", () => { + const base = new LoroDoc(); + base.setPeerId(1); + const baseList = base.getMovableList("list"); + for (const value of ["a", "b", "c", "d"]) + baseList.push(value); + base.commit(); + const left = base.fork(); + left.setPeerId(2); + left.getMovableList("list").move(0, 3); + left.commit(); + const leftFrontiers = left.frontiers(); + const right = base.fork(); + right.setPeerId(3); + right.getMovableList("list").move(3, 0); + right.commit(); + const rightFrontiers = right.frontiers(); + left.import(right.export({ mode: "update" })); + const mergedFrontiers = left.frontiers(); + const list = left.getMovableList("list"); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + leftFrontiers, + rightFrontiers, + mergedFrontiers, + rightFrontiers, + leftFrontiers, + mergedFrontiers, + ]) { + const expected = left.forkAt(frontiers).getMovableList("list").toJSON(); + left.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("switches concurrent movable-list branches with inserted and deleted items", () => { + const base = new LoroDoc(); + base.setPeerId(1); + const baseList = base.getMovableList("list"); + for (const value of ["a", "b", "c", "d"]) + baseList.push(value); + base.commit(); + const left = base.fork(); + left.setPeerId(2); + const leftList = left.getMovableList("list"); + leftList.move(0, 3); + leftList.insert(1, "L"); + left.commit(); + const leftFrontiers = left.frontiers(); + const right = base.fork(); + right.setPeerId(3); + const rightList = right.getMovableList("list"); + rightList.move(3, 0); + rightList.delete(2, 1); + right.commit(); + const rightFrontiers = right.frontiers(); + left.import(right.export({ mode: "update", from: base.oplogVersion() })); + const mergedFrontiers = left.frontiers(); + const list = left.getMovableList("list"); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + leftFrontiers, + rightFrontiers, + mergedFrontiers, + rightFrontiers, + leftFrontiers, + mergedFrontiers, + ]) { + const expected = left.forkAt(frontiers).getMovableList("list").toJSON(); + left.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + }); + test("matches rebuilt movable-list branches across randomized concurrent moves", () => { + let random = 2048475843; + const nextRandom = () => { + random ^= random << 13; + random ^= random >>> 17; + random ^= random << 5; + return random >>> 0; + }; + for (let scenario = 0; scenario < 24; scenario += 1) { + const base = new LoroDoc(); + base.setPeerId(1); + const baseList = base.getMovableList("list"); + for (let index = 0; index < 8; index += 1) + baseList.push(index); + base.commit(); + const left = base.fork(); + left.setPeerId(2); + const leftList = left.getMovableList("list"); + for (let move = 0; move < 4; move += 1) { + const from = nextRandom() % leftList.length; + const to = (from + 1 + (nextRandom() % (leftList.length - 1))) % leftList.length; + leftList.move(from, to); + left.commit(); + } + const leftFrontiers = left.frontiers(); + const right = base.fork(); + right.setPeerId(3); + const rightList = right.getMovableList("list"); + for (let move = 0; move < 4; move += 1) { + const from = nextRandom() % rightList.length; + const to = (from + 1 + (nextRandom() % (rightList.length - 1))) % rightList.length; + rightList.move(from, to); + right.commit(); + } + const rightFrontiers = right.frontiers(); + const importMirror = left.fork(); + let importBatch; + const unsubscribe = left.subscribe((event) => { + importBatch = event; + }); + left.import(right.export({ mode: "update", from: base.oplogVersion() })); + unsubscribe(); + const mergedFrontiers = left.frontiers(); + const list = left.getMovableList("list"); + expect(importBatch).toBeDefined(); + importMirror.applyDiff(importBatch.events.map(({ target, diff }) => [target, diff])); + expect(importMirror.getMovableList("list").toJSON()).toEqual(list.toJSON()); + expect(list.toJSON()).toEqual(left.forkAt(mergedFrontiers).getMovableList("list").toJSON()); + const reset = vi.spyOn(list, "_reset"); + for (const frontiers of [ + leftFrontiers, + rightFrontiers, + mergedFrontiers, + leftFrontiers, + mergedFrontiers, + rightFrontiers, + ]) { + const expected = left.forkAt(frontiers).getMovableList("list").toJSON(); + left.checkout(frontiers); + expect(list.toJSON()).toEqual(expected); + } + expect(reset).not.toHaveBeenCalled(); + const expectedRight = left.forkAt(rightFrontiers).getMovableList("list").toJSON(); + const mirror = left.forkAt(leftFrontiers); + mirror.applyDiff(left.diff(leftFrontiers, rightFrontiers, false)); + expect(mirror.getMovableList("list").toJSON()).toEqual(expectedRight); + expect(reset).not.toHaveBeenCalled(); + } + }); + test("switches a large contiguous text deletion without rebuilding the text", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const text = doc.getText("text"); + const value = "x".repeat(16384); + text.insert(0, value); + doc.commit(); + const beforeDelete = doc.frontiers(); + text.delete(0, value.length); + doc.commit(); + expect(text.toString()).toBe(""); + doc.checkout(beforeDelete); + expect(text.toString()).toBe(value); + const mirror = doc.forkAt(beforeDelete); + let batch; + const unsubscribe = doc.subscribe((event) => { + batch = event; + }); + doc.checkoutToLatest(); + unsubscribe(); + expect(text.toString()).toBe(""); + expect(batch).toBeDefined(); + expect(JSON.stringify(batch.events).length).toBeLessThan(256); + mirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); + expect(mirror.getText("text").toString()).toBe(""); + }); + test("keeps overlapping concurrent range deletions during branch switches", () => { + const base = new LoroDoc(); + base.setPeerId(1); + base.getText("text").insert(0, "x".repeat(1024)); + base.commit(); + const baseFrontiers = base.frontiers(); + const left = base.fork(); + left.setPeerId(2); + left.getText("text").delete(0, 1024); + left.commit(); + const leftFrontiers = left.frontiers(); + const right = base.fork(); + right.setPeerId(3); + right.getText("text").delete(200, 600); + right.commit(); + const rightFrontiers = right.frontiers(); + left.import(right.export({ mode: "update", from: base.oplogVersion() })); + left.checkout(rightFrontiers); + expect(left.getText("text").length).toBe(424); + left.checkout(baseFrontiers); + expect(left.getText("text").length).toBe(1024); + left.checkout(leftFrontiers); + expect(left.getText("text").length).toBe(0); + }); +}); diff --git a/loro-js/vite.config.js b/loro-js/vite.config.js new file mode 100644 index 000000000..aaef39404 --- /dev/null +++ b/loro-js/vite.config.js @@ -0,0 +1,39 @@ +import { defineConfig } from "vite-plus"; +export default defineConfig({ + fmt: { + ignorePatterns: ["dist/**", "node_modules/**", "tests/fixtures/**"], + printWidth: 90, + semi: true, + singleQuote: false, + sortPackageJson: false, + trailingComma: "all", + }, + lint: { + ignorePatterns: ["dist/**", "node_modules/**", "tests/fixtures/**"], + options: { + denyWarnings: true, + reportUnusedDisableDirectives: "error", + typeAware: true, + typeCheck: true, + }, + plugins: ["typescript", "oxc", "import", "vitest"], + rules: { + "no-console": "error", + }, + }, + pack: { + clean: true, + dts: true, + entry: ["src/index.ts", "src/codec/index.ts"], + format: "esm", + outDir: "dist", + platform: "neutral", + sourcemap: true, + target: "es2022", + tsconfig: "tsconfig.build.json", + }, + test: { + environment: "node", + include: ["tests/**/*.test.ts"], + }, +}); From a7b87471df029f763675616a8554ba661dd28c52 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 20 Jul 2026 01:52:19 +0800 Subject: [PATCH 4/5] chore: remove generated loro-js JavaScript --- loro-js/src/codec/bytes.js | 235 - loro-js/src/codec/change-block-codec.js | 789 -- loro-js/src/codec/change-block-tables.js | 375 - loro-js/src/codec/change-block.js | 38 - loro-js/src/codec/change-value.js | 375 - loro-js/src/codec/container-id.js | 225 - loro-js/src/codec/document.js | 115 - loro-js/src/codec/errors.js | 24 - loro-js/src/codec/id.js | 53 - loro-js/src/codec/index.js | 20 - loro-js/src/codec/leb128.js | 149 - loro-js/src/codec/lz4.js | 307 - loro-js/src/codec/position-arena.js | 67 - loro-js/src/codec/postcard.js | 172 - loro-js/src/codec/serde-columnar.js | 902 --- loro-js/src/codec/sstable.js | 261 - loro-js/src/codec/state-snapshot.js | 731 -- loro-js/src/codec/types.js | 8 - loro-js/src/codec/value.js | 156 - loro-js/src/codec/version.js | 60 - loro-js/src/codec/xxhash32.js | 68 - loro-js/src/index.js | 1 - loro-js/src/runtime/containers.js | 2597 ------- loro-js/src/runtime/document.js | 6813 ----------------- loro-js/src/runtime/ephemeral.js | 560 -- loro-js/src/runtime/event-diff.js | 371 - loro-js/src/runtime/fractional-index.js | 92 - loro-js/src/runtime/ids.js | 151 - loro-js/src/runtime/index.js | 8 - loro-js/src/runtime/json-updates.js | 60 - loro-js/src/runtime/jsonpath.js | 833 -- loro-js/src/runtime/mergeable.js | 124 - loro-js/src/runtime/ordered-index.js | 303 - .../src/runtime/sequence-deletion-index.js | 227 - loro-js/src/runtime/sequence-index.js | 3173 -------- loro-js/src/runtime/text-style-index.js | 336 - loro-js/src/runtime/types.js | 1 - loro-js/src/runtime/undo.js | 274 - loro-js/src/runtime/version-vector.js | 112 - loro-js/tests/change-block-codec.test.js | 133 - loro-js/tests/change-block-tables.test.js | 69 - loro-js/tests/change-value.test.js | 106 - loro-js/tests/document.test.js | 40 - loro-js/tests/event-diff.test.js | 290 - loro-js/tests/fugue-index.test.js | 21 - loro-js/tests/indexes.test.js | 620 -- loro-js/tests/lz4.test.js | 65 - loro-js/tests/primitives.test.js | 54 - loro-js/tests/runtime.test.js | 1708 ----- loro-js/tests/rust-interop.test.js | 176 - loro-js/tests/semantic-codecs.test.js | 110 - loro-js/tests/serde-columnar.test.js | 84 - loro-js/tests/sstable.test.js | 33 - loro-js/tests/state-snapshot.test.js | 141 - loro-js/tests/version-transition.test.js | 790 -- loro-js/vite.config.js | 39 - 56 files changed, 25645 deletions(-) delete mode 100644 loro-js/src/codec/bytes.js delete mode 100644 loro-js/src/codec/change-block-codec.js delete mode 100644 loro-js/src/codec/change-block-tables.js delete mode 100644 loro-js/src/codec/change-block.js delete mode 100644 loro-js/src/codec/change-value.js delete mode 100644 loro-js/src/codec/container-id.js delete mode 100644 loro-js/src/codec/document.js delete mode 100644 loro-js/src/codec/errors.js delete mode 100644 loro-js/src/codec/id.js delete mode 100644 loro-js/src/codec/index.js delete mode 100644 loro-js/src/codec/leb128.js delete mode 100644 loro-js/src/codec/lz4.js delete mode 100644 loro-js/src/codec/position-arena.js delete mode 100644 loro-js/src/codec/postcard.js delete mode 100644 loro-js/src/codec/serde-columnar.js delete mode 100644 loro-js/src/codec/sstable.js delete mode 100644 loro-js/src/codec/state-snapshot.js delete mode 100644 loro-js/src/codec/types.js delete mode 100644 loro-js/src/codec/value.js delete mode 100644 loro-js/src/codec/version.js delete mode 100644 loro-js/src/codec/xxhash32.js delete mode 100644 loro-js/src/index.js delete mode 100644 loro-js/src/runtime/containers.js delete mode 100644 loro-js/src/runtime/document.js delete mode 100644 loro-js/src/runtime/ephemeral.js delete mode 100644 loro-js/src/runtime/event-diff.js delete mode 100644 loro-js/src/runtime/fractional-index.js delete mode 100644 loro-js/src/runtime/ids.js delete mode 100644 loro-js/src/runtime/index.js delete mode 100644 loro-js/src/runtime/json-updates.js delete mode 100644 loro-js/src/runtime/jsonpath.js delete mode 100644 loro-js/src/runtime/mergeable.js delete mode 100644 loro-js/src/runtime/ordered-index.js delete mode 100644 loro-js/src/runtime/sequence-deletion-index.js delete mode 100644 loro-js/src/runtime/sequence-index.js delete mode 100644 loro-js/src/runtime/text-style-index.js delete mode 100644 loro-js/src/runtime/types.js delete mode 100644 loro-js/src/runtime/undo.js delete mode 100644 loro-js/src/runtime/version-vector.js delete mode 100644 loro-js/tests/change-block-codec.test.js delete mode 100644 loro-js/tests/change-block-tables.test.js delete mode 100644 loro-js/tests/change-value.test.js delete mode 100644 loro-js/tests/document.test.js delete mode 100644 loro-js/tests/event-diff.test.js delete mode 100644 loro-js/tests/fugue-index.test.js delete mode 100644 loro-js/tests/indexes.test.js delete mode 100644 loro-js/tests/lz4.test.js delete mode 100644 loro-js/tests/primitives.test.js delete mode 100644 loro-js/tests/runtime.test.js delete mode 100644 loro-js/tests/rust-interop.test.js delete mode 100644 loro-js/tests/semantic-codecs.test.js delete mode 100644 loro-js/tests/serde-columnar.test.js delete mode 100644 loro-js/tests/sstable.test.js delete mode 100644 loro-js/tests/state-snapshot.test.js delete mode 100644 loro-js/tests/version-transition.test.js delete mode 100644 loro-js/vite.config.js diff --git a/loro-js/src/codec/bytes.js b/loro-js/src/codec/bytes.js deleted file mode 100644 index 518dded7e..000000000 --- a/loro-js/src/codec/bytes.js +++ /dev/null @@ -1,235 +0,0 @@ -import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; -const U8_MAX = 0xff; -const U16_MAX = 0xffff; -const U32_MAX = 4294967295; -const U64_MAX = 0xffffffffffffffffn; -export class ByteReader { - bytes; - #offset; - #end; - constructor(bytes, offset = 0, length = bytes.length - offset) { - decodeAssert(Number.isSafeInteger(offset) && offset >= 0, "invalid reader offset"); - decodeAssert(Number.isSafeInteger(length) && length >= 0, "invalid reader length"); - decodeAssert(offset + length <= bytes.length, "reader range is out of bounds"); - this.bytes = bytes; - this.#offset = offset; - this.#end = offset + length; - } - get position() { - return this.#offset; - } - get remaining() { - return this.#end - this.#offset; - } - readU8() { - this.require(1); - const value = this.bytes[this.#offset]; - this.#offset += 1; - return value; - } - readU16LE() { - const a = this.readU8(); - const b = this.readU8(); - return a | (b << 8); - } - readU16BE() { - const a = this.readU8(); - const b = this.readU8(); - return (a << 8) | b; - } - readU32LE() { - const a = this.readU8(); - const b = this.readU8(); - const c = this.readU8(); - const d = this.readU8(); - return (a | (b << 8) | (c << 16) | (d << 24)) >>> 0; - } - readU32BE() { - const a = this.readU8(); - const b = this.readU8(); - const c = this.readU8(); - const d = this.readU8(); - return ((a << 24) | (b << 16) | (c << 8) | d) >>> 0; - } - readU64LE() { - let value = 0n; - for (let shift = 0n; shift < 64n; shift += 8n) { - value |= BigInt(this.readU8()) << shift; - } - return value; - } - readU64BE() { - let value = 0n; - for (let i = 0; i < 8; i += 1) { - value = (value << 8n) | BigInt(this.readU8()); - } - return value; - } - readBytes(length) { - this.require(length); - const start = this.#offset; - this.#offset += length; - return this.bytes.subarray(start, this.#offset); - } - readRemaining() { - return this.readBytes(this.remaining); - } - skip(length) { - this.require(length); - this.#offset += length; - } - assertEnd(message = "trailing bytes") { - if (this.remaining !== 0) { - throw new LoroDecodeError(message, this.#offset); - } - } - require(length) { - if (!Number.isSafeInteger(length) || length < 0) { - throw new LoroDecodeError("invalid read length", this.#offset); - } - if (length > this.remaining) { - throw new LoroDecodeError("unexpected end of input", this.#offset); - } - } -} -export class ByteWriter { - #buffer; - #length = 0; - constructor(initialCapacity = 64) { - encodeAssert(Number.isSafeInteger(initialCapacity) && initialCapacity >= 0, "invalid writer capacity"); - this.#buffer = new Uint8Array(initialCapacity); - } - get length() { - return this.#length; - } - writeU8(value) { - assertUnsignedNumber(value, U8_MAX, "u8"); - this.ensureCapacity(1); - this.#buffer[this.#length] = value; - this.#length += 1; - } - writeU16LE(value) { - assertUnsignedNumber(value, U16_MAX, "u16"); - this.writeU8(value & U8_MAX); - this.writeU8(value >>> 8); - } - writeU16BE(value) { - assertUnsignedNumber(value, U16_MAX, "u16"); - this.writeU8(value >>> 8); - this.writeU8(value & U8_MAX); - } - writeU32LE(value) { - assertUnsignedNumber(value, U32_MAX, "u32"); - this.writeU8(value & U8_MAX); - this.writeU8((value >>> 8) & U8_MAX); - this.writeU8((value >>> 16) & U8_MAX); - this.writeU8((value >>> 24) & U8_MAX); - } - writeU32BE(value) { - assertUnsignedNumber(value, U32_MAX, "u32"); - this.writeU8((value >>> 24) & U8_MAX); - this.writeU8((value >>> 16) & U8_MAX); - this.writeU8((value >>> 8) & U8_MAX); - this.writeU8(value & U8_MAX); - } - writeU64LE(value) { - assertUnsignedBigInt(value, U64_MAX, "u64"); - for (let shift = 0n; shift < 64n; shift += 8n) { - this.writeU8(Number((value >> shift) & 0xffn)); - } - } - writeU64BE(value) { - assertUnsignedBigInt(value, U64_MAX, "u64"); - for (let shift = 56n; shift >= 0n; shift -= 8n) { - this.writeU8(Number((value >> shift) & 0xffn)); - } - } - writeBytes(bytes) { - this.ensureCapacity(bytes.length); - this.#buffer.set(bytes, this.#length); - this.#length += bytes.length; - } - toUint8Array() { - // When the buffer is exactly full, hand it over instead of copying; any - // later write would reallocate, so the returned array stays intact. This - // matches Lz4Output.finish in lz4.ts. - return this.#length === this.#buffer.length - ? this.#buffer - : this.#buffer.slice(0, this.#length); - } - ensureCapacity(extra) { - const required = this.#length + extra; - if (required <= this.#buffer.length) { - return; - } - let capacity = Math.max(64, this.#buffer.length); - while (capacity < required) { - capacity = Math.max(required, capacity * 2); - } - const next = new Uint8Array(capacity); - next.set(this.#buffer.subarray(0, this.#length)); - this.#buffer = next; - } -} -function assertUnsignedNumber(value, max, label) { - if (!Number.isSafeInteger(value) || value < 0 || value > max) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } -} -function assertUnsignedBigInt(value, max, label) { - if (value < 0n || value > max) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } -} -export function concatBytes(...parts) { - let length = 0; - for (const part of parts) { - length += part.length; - encodeAssert(Number.isSafeInteger(length), "byte sequence is too large"); - } - const output = new Uint8Array(length); - let offset = 0; - for (const part of parts) { - output.set(part, offset); - offset += part.length; - } - return output; -} -export function bytesEqual(a, b) { - if (a.length !== b.length) { - return false; - } - for (let i = 0; i < a.length; i += 1) { - if (a[i] !== b[i]) { - return false; - } - } - return true; -} -export function compareBytes(a, b) { - const length = Math.min(a.length, b.length); - for (let i = 0; i < length; i += 1) { - const diff = a[i] - b[i]; - if (diff !== 0) { - return diff; - } - } - return a.length - b.length; -} -export function bytesToHex(bytes) { - let output = ""; - for (const byte of bytes) { - output += byte.toString(16).padStart(2, "0"); - } - return output; -} -export function hexToBytes(hex) { - if (hex.length % 2 !== 0 || !/^[0-9a-f]*$/iu.test(hex)) { - throw new LoroDecodeError("invalid hexadecimal byte string"); - } - const output = new Uint8Array(hex.length / 2); - for (let i = 0; i < output.length; i += 1) { - output[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); - } - return output; -} diff --git a/loro-js/src/codec/change-block-codec.js b/loro-js/src/codec/change-block-codec.js deleted file mode 100644 index 3afef5b2b..000000000 --- a/loro-js/src/codec/change-block-codec.js +++ /dev/null @@ -1,789 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { decodeChangeKeys, decodeChangesHeader, decodeChangesMetadata, decodeContainerArena, decodeDeleteStartIdColumns, decodeEncodedOperationColumns, encodeChangeKeys, encodeChangesHeader, encodeChangesMetadata, encodeContainerArena, encodeDeleteStartIdColumns, encodeEncodedOperationColumns, } from "./change-block-tables"; -import { decodeEncodedChangeBlock, encodeEncodedChangeBlock } from "./change-block"; -import { readChangeValueContent, encodeChangeValueContent, } from "./change-value"; -import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; -import { decodePositionArena, encodePositionArena } from "./position-arena"; -import { ContainerType, } from "./types"; -const DELETED_TREE_ROOT = { - peer: 0xffffffffffffffffn, - counter: 2147483647, -}; -function decodeChangeBlockParts(bytes) { - const encoded = decodeEncodedChangeBlock(bytes); - const header = decodeChangesHeader(encoded.header, { - changeCount: encoded.changeCount, - counterStart: encoded.counterStart, - counterLength: encoded.counterLength, - lamportStart: encoded.lamportStart, - lamportLength: encoded.lamportLength, - }); - const metadata = decodeChangesMetadata(encoded.changeMetadata, encoded.changeCount); - const keys = decodeChangeKeys(encoded.keys); - const containers = decodeContainerArena(encoded.containerIds, header.peers, keys); - const positions = decodePositionArena(encoded.positions); - const operations = decodeEncodedOperationColumns(encoded.operations); - const context = { - peers: header.peers, - keys, - positions, - deleteStartIds: decodeDeleteStartIdColumns(encoded.deleteStartIds), - deleteIndex: 0, - }; - return { encoded, header, metadata, keys, containers, positions, operations, context }; -} -export function decodeChangeBlock(bytes) { - const { encoded, header, metadata, keys, containers, positions, operations, context } = decodeChangeBlockParts(bytes); - const mutableOperations = Array.from({ length: encoded.changeCount }, () => []); - let counter = encoded.counterStart | 0; - let changeIndex = 0; - const valueReader = new ByteReader(encoded.values); - const operationId = { peer: header.peer, counter }; - for (let row = 0; row < operations.containerIndices.length; row += 1) { - const length = operations.lengths[row]; - decodeAssert(length > 0 && length <= 2147483647, "invalid operation length"); - const containerIndex = operations.containerIndices[row]; - decodeAssert(containerIndex < containers.length, "invalid operation container index"); - const container = containers[containerIndex]; - const value = readChangeValueContent(operations.valueTypes[row], valueReader); - operationId.counter = counter; - const content = decodeOperationContent(container, operations.properties[row], value, operationId, context); - mutableOperations[changeIndex].push({ - container, - counter, - length, - content, - }); - counter = checkedCounter(counter + length); - const nextBoundary = header.counters[changeIndex + 1]; - decodeAssert(nextBoundary !== undefined, "operation exceeds change boundaries"); - decodeAssert(counter <= nextBoundary, "operation crosses a change boundary"); - if (counter === nextBoundary && changeIndex + 1 < encoded.changeCount) { - changeIndex += 1; - } - } - decodeAssert(valueReader.remaining === 0, "trailing change value bytes"); - decodeAssert(context.deleteIndex === context.deleteStartIds.peerIndices.length, "unused delete start IDs"); - decodeAssert(counter === (encoded.counterStart | 0) + encoded.counterLength, "operation lengths do not fill the block counter range"); - const changes = []; - for (let index = 0; index < encoded.changeCount; index += 1) { - changes.push({ - id: { peer: header.peer, counter: header.counters[index] }, - timestamp: metadata.timestamps[index], - dependencies: header.dependencies[index], - lamport: header.lamports[index], - message: metadata.commitMessages[index], - operations: mutableOperations[index], - }); - } - return { peers: header.peers, keys, containers, positions, changes }; -} -/** Fully validates a block while retaining only its covered counter range. */ -export function validateChangeBlock(bytes) { - const { encoded, header, containers, operations, context } = decodeChangeBlockParts(bytes); - let counter = encoded.counterStart | 0; - let changeIndex = 0; - const valueReader = new ByteReader(encoded.values); - const operationId = { peer: header.peer, counter }; - for (let row = 0; row < operations.containerIndices.length; row += 1) { - const length = operations.lengths[row]; - decodeAssert(length > 0 && length <= 2147483647, "invalid operation length"); - const containerIndex = operations.containerIndices[row]; - decodeAssert(containerIndex < containers.length, "invalid operation container index"); - const value = readChangeValueContent(operations.valueTypes[row], valueReader); - operationId.counter = counter; - decodeOperationContent(containers[containerIndex], operations.properties[row], value, operationId, context, false); - counter = checkedCounter(counter + length); - const nextBoundary = header.counters[changeIndex + 1]; - decodeAssert(nextBoundary !== undefined, "operation exceeds change boundaries"); - decodeAssert(counter <= nextBoundary, "operation crosses a change boundary"); - if (counter === nextBoundary && changeIndex + 1 < encoded.changeCount) { - changeIndex += 1; - } - } - decodeAssert(valueReader.remaining === 0, "trailing change value bytes"); - decodeAssert(context.deleteIndex === context.deleteStartIds.peerIndices.length, "unused delete start IDs"); - const counterEnd = checkedCounter((encoded.counterStart | 0) + encoded.counterLength); - decodeAssert(counter === counterEnd, "operation lengths do not fill the block counter range"); - return { peer: header.peer, counterStart: encoded.counterStart | 0, counterEnd }; -} -export function encodeChangeBlock(block) { - if (block.changes.length === 0) { - throw new LoroEncodeError("cannot encode an empty change block"); - } - const firstPeer = block.changes[0].id.peer; - const tables = initializeTables(block); - if (tables.peers.length === 0) { - registerPeer(tables, firstPeer); - } - else if (tables.peers[0] !== firstPeer) { - throw new LoroEncodeError("the first peer table entry must be the block peer"); - } - const operationContainerIndices = []; - const operationProperties = []; - const operationValueTypes = []; - const operationLengths = []; - const deleteStartIds = { - peerIndices: [], - counters: [], - lengths: [], - }; - const valueWriter = new ByteWriter(); - const lengths = []; - const counters = []; - let expectedCounter = block.changes[0].id.counter; - for (const change of block.changes) { - if (change.id.peer !== firstPeer || change.id.counter !== expectedCounter) { - throw new LoroEncodeError("changes must be continuous and belong to one peer"); - } - counters.push(change.id.counter); - let atomLength = 0; - for (const operation of change.operations) { - if (operation.counter !== expectedCounter + atomLength) { - throw new LoroEncodeError("operation counters are not continuous"); - } - assertOperationLength(operation.length); - const containerIndex = registerContainer(tables, operation.container); - const encodedContent = encodeOperationContent(operation.content, tables, deleteStartIds); - const encodedValue = encodeChangeValueContent(encodedContent.value); - valueWriter.writeBytes(encodedValue.bytes); - operationContainerIndices.push(containerIndex); - operationProperties.push(encodedContent.property); - operationValueTypes.push(encodedValue.tag); - operationLengths.push(operation.length); - atomLength += operation.length; - if (atomLength > 2147483647) { - throw new LoroEncodeError("change atom length is out of range"); - } - } - if (atomLength === 0) { - throw new LoroEncodeError("changes must contain at least one operation atom"); - } - lengths.push(atomLength); - expectedCounter = checkedEncodeCounter(expectedCounter + atomLength); - } - counters.push(expectedCounter); - for (const container of tables.containers) { - if (container.kind === "root") { - registerKey(tables, container.name); - } - else { - registerPeer(tables, container.peer); - } - } - for (const change of block.changes) { - for (const dependency of change.dependencies) { - registerPeer(tables, dependency.peer); - } - } - const header = { - peer: firstPeer, - peers: tables.peers, - counters, - lengths, - lamports: block.changes.map((change) => change.lamport), - dependencies: block.changes.map((change) => change.dependencies), - }; - const counterStart = block.changes[0].id.counter; - if (counterStart < 0) { - throw new LoroEncodeError("change block counter start must be nonnegative"); - } - const counterLength = lengths.reduce((sum, length) => sum + length, 0); - const lamportStart = block.changes[0].lamport; - const finalChange = block.changes[block.changes.length - 1]; - const lamportEnd = finalChange.lamport + lengths[lengths.length - 1]; - const lamportLength = lamportEnd - lamportStart; - if (!Number.isSafeInteger(lamportStart) || - lamportStart < 0 || - lamportStart > 4294967295 || - lamportLength < 0 || - lamportLength > 4294967295) { - throw new LoroEncodeError("change block lamport range is invalid"); - } - return encodeEncodedChangeBlock({ - counterStart, - counterLength, - lamportStart, - lamportLength, - changeCount: block.changes.length, - header: encodeChangesHeader(header), - changeMetadata: encodeChangesMetadata({ - timestamps: block.changes.map((change) => change.timestamp), - commitMessages: block.changes.map((change) => change.message), - }), - containerIds: encodeContainerArena(tables.containers, tables.peers, tables.keys), - keys: encodeChangeKeys(tables.keys), - positions: encodePositionArena(tables.positions), - operations: encodeEncodedOperationColumns({ - containerIndices: operationContainerIndices, - properties: operationProperties, - valueTypes: operationValueTypes, - lengths: operationLengths, - }), - deleteStartIds: encodeDeleteStartIdColumns(deleteStartIds), - values: valueWriter.toUint8Array(), - }); -} -/** - * Encodes a block holding exactly one change, the shape documents emit per - * history record. Seeds the peer/key tables directly instead of copying the - * wrapper arrays initializeTables accepts, avoiding one block object and two - * single-element arrays per record. The bytes are identical to - * encodeChangeBlock with `{ peers: [change.id.peer], keys, containers: [], - * positions: [], changes: [change] }`; keep the shared encoding steps below in - * sync with it. - */ -export function encodeSingleChangeBlock(change, keys) { - const peer = change.id.peer; - const copiedKeys = [...keys]; - const keyIndices = new Map(); - for (let index = 0; index < copiedKeys.length; index += 1) { - // The first occurrence wins, matching initializeTables. - if (!keyIndices.has(copiedKeys[index])) { - keyIndices.set(copiedKeys[index], index); - } - } - const tables = { - peers: [peer], - peerIndices: new Map([[peer, 0]]), - keys: copiedKeys, - keyIndices, - containers: [], - rootContainerIndices: new Map(), - normalContainerIndices: new Map(), - positions: [], - positionIndices: new Map(), - }; - const operationContainerIndices = []; - const operationProperties = []; - const operationValueTypes = []; - const operationLengths = []; - const deleteStartIds = { - peerIndices: [], - counters: [], - lengths: [], - }; - const valueWriter = new ByteWriter(); - const counterStart = change.id.counter; - let atomLength = 0; - for (const operation of change.operations) { - if (operation.counter !== counterStart + atomLength) { - throw new LoroEncodeError("operation counters are not continuous"); - } - assertOperationLength(operation.length); - const containerIndex = registerContainer(tables, operation.container); - const encodedContent = encodeOperationContent(operation.content, tables, deleteStartIds); - const encodedValue = encodeChangeValueContent(encodedContent.value); - valueWriter.writeBytes(encodedValue.bytes); - operationContainerIndices.push(containerIndex); - operationProperties.push(encodedContent.property); - operationValueTypes.push(encodedValue.tag); - operationLengths.push(operation.length); - atomLength += operation.length; - if (atomLength > 2147483647) { - throw new LoroEncodeError("change atom length is out of range"); - } - } - if (atomLength === 0) { - throw new LoroEncodeError("changes must contain at least one operation atom"); - } - const counterEnd = checkedEncodeCounter(counterStart + atomLength); - for (const container of tables.containers) { - if (container.kind === "root") { - registerKey(tables, container.name); - } - else { - registerPeer(tables, container.peer); - } - } - for (const dependency of change.dependencies) { - registerPeer(tables, dependency.peer); - } - if (counterStart < 0) { - throw new LoroEncodeError("change block counter start must be nonnegative"); - } - const lamportStart = change.lamport; - if (!Number.isSafeInteger(lamportStart) || - lamportStart < 0 || - lamportStart > 4294967295) { - throw new LoroEncodeError("change block lamport range is invalid"); - } - return encodeEncodedChangeBlock({ - counterStart, - counterLength: atomLength, - lamportStart, - lamportLength: atomLength, - changeCount: 1, - header: encodeChangesHeader({ - peer, - peers: tables.peers, - counters: [counterStart, counterEnd], - lengths: [atomLength], - lamports: [lamportStart], - dependencies: [change.dependencies], - }), - changeMetadata: encodeChangesMetadata({ - timestamps: [change.timestamp], - commitMessages: [change.message], - }), - containerIds: encodeContainerArena(tables.containers, tables.peers, tables.keys), - keys: encodeChangeKeys(tables.keys), - positions: encodePositionArena(tables.positions), - operations: encodeEncodedOperationColumns({ - containerIndices: operationContainerIndices, - properties: operationProperties, - valueTypes: operationValueTypes, - lengths: operationLengths, - }), - deleteStartIds: encodeDeleteStartIdColumns(deleteStartIds), - values: valueWriter.toUint8Array(), - }); -} -function decodeOperationContent(container, property, value, operationId, context, materialize = true) { - const type = container.containerType; - if (type === ContainerType.Map) { - const key = getIndex(context.keys, property, "map key"); - if (value.type === "delete-once") { - return materialize ? { type: "map-delete", key } : undefined; - } - if (value.type === "loro-value") { - return materialize ? { type: "map-insert", key, value: value.value } : undefined; - } - throw new LoroDecodeError("invalid map operation value"); - } - if (type === ContainerType.Text) { - if (value.type === "string") { - decodeAssert(property >= 0, "invalid text insertion position"); - return materialize - ? { type: "text-insert", position: property, value: value.value } - : undefined; - } - if (value.type === "delete-sequence") { - const deletion = takeDeletion(context, materialize); - return materialize - ? { - type: "text-delete", - position: property, - length: deletion.length, - startId: deletion.startId, - } - : undefined; - } - if (value.type === "mark-start") { - decodeAssert(property >= 0, "invalid text mark position"); - const length = bigintToIndex(value.length, 4294967295, "text mark length"); - const keyIndex = bigintToIndex(value.keyIndex, context.keys.length - 1, "text mark key index"); - const end = property + length; - decodeAssert(end <= 4294967295, "text mark end is out of range"); - return materialize - ? { - type: "text-mark", - start: property, - end, - key: context.keys[keyIndex], - value: value.value, - info: value.info, - } - : undefined; - } - if (value.type === "null") { - return materialize ? { type: "text-mark-end" } : undefined; - } - throw new LoroDecodeError("invalid text operation value"); - } - if (type === ContainerType.List) { - if (value.type === "loro-value" && value.value.type === "list") { - decodeAssert(property >= 0, "invalid list insertion position"); - return materialize - ? { type: "list-insert", position: property, values: value.value.value } - : undefined; - } - if (value.type === "delete-sequence") { - const deletion = takeDeletion(context, materialize); - return materialize - ? { - type: "list-delete", - position: property, - length: deletion.length, - startId: deletion.startId, - } - : undefined; - } - throw new LoroDecodeError("invalid list operation value"); - } - if (type === ContainerType.MovableList) { - if (value.type === "loro-value" && value.value.type === "list") { - decodeAssert(property >= 0, "invalid movable-list insertion position"); - return materialize - ? { - type: "movable-list-insert", - position: property, - values: value.value.value, - } - : undefined; - } - if (value.type === "delete-sequence") { - const deletion = takeDeletion(context, materialize); - return materialize - ? { - type: "movable-list-delete", - position: property, - length: deletion.length, - startId: deletion.startId, - } - : undefined; - } - if (value.type === "list-move") { - decodeAssert(property >= 0, "invalid movable-list destination"); - const from = bigintToIndex(value.from, 4294967295, "movable-list source"); - const peer = context.peers[bigintToIndex(value.fromPeerIndex, context.peers.length - 1, "movable-list peer index")]; - const lamport = bigintToIndex(value.lamport, 4294967295, "movable-list lamport"); - return materialize - ? { - type: "movable-list-move", - from, - to: property, - elementId: { peer, lamport }, - } - : undefined; - } - if (value.type === "list-set") { - const peer = context.peers[bigintToIndex(value.peerIndex, context.peers.length - 1, "movable-list peer index")]; - return materialize - ? { - type: "movable-list-set", - elementId: { peer, lamport: value.lamport }, - value: value.value, - } - : undefined; - } - throw new LoroDecodeError("invalid movable-list operation value"); - } - if (type === ContainerType.Tree) { - if (value.type !== "raw-tree-move") { - throw new LoroDecodeError("invalid tree operation value"); - } - const subjectPeer = context.peers[bigintToIndex(value.subjectPeerIndex, context.peers.length - 1, "tree subject peer index")]; - const parentPeer = value.parentIsNull - ? undefined - : context.peers[bigintToIndex(value.parentPeerIndex, context.peers.length - 1, "tree parent peer index")]; - const deleting = parentPeer === DELETED_TREE_ROOT.peer && - value.parentCounter === DELETED_TREE_ROOT.counter; - if (deleting) { - return materialize - ? { - type: "tree-delete", - subject: { peer: subjectPeer, counter: value.subjectCounter }, - } - : undefined; - } - const position = context.positions[bigintToIndex(value.positionIndex, context.positions.length - 1, "tree position index")]; - if (!materialize) - return undefined; - const subject = { peer: subjectPeer, counter: value.subjectCounter }; - const parent = parentPeer === undefined - ? undefined - : { peer: parentPeer, counter: value.parentCounter }; - return idsEqual(subject, operationId) - ? { type: "tree-create", subject, parent, position } - : { type: "tree-move", subject, parent, position }; - } - return materialize ? { type: "future", property, value } : undefined; -} -function encodeOperationContent(content, tables, deletions) { - switch (content.type) { - case "map-insert": - return { - property: registerKey(tables, content.key), - value: { type: "loro-value", value: content.value }, - }; - case "map-delete": - return { - property: registerKey(tables, content.key), - value: { type: "delete-once" }, - }; - case "text-insert": - return { - property: assertNonnegativePosition(content.position), - value: { type: "string", value: content.value }, - }; - case "text-delete": - registerDeletion(deletions, tables, content.startId, content.length); - return { property: content.position, value: { type: "delete-sequence" } }; - case "text-mark": { - const length = content.end - content.start; - if (length < 0) { - throw new LoroEncodeError("text mark end precedes its start"); - } - return { - property: assertNonnegativePosition(content.start), - value: { - type: "mark-start", - info: content.info, - length: BigInt(length), - keyIndex: BigInt(registerKey(tables, content.key)), - value: content.value, - }, - }; - } - case "text-mark-end": - return { property: 0, value: { type: "null" } }; - case "list-insert": - return { - property: assertNonnegativePosition(content.position), - value: { type: "loro-value", value: { type: "list", value: content.values } }, - }; - case "list-delete": - registerDeletion(deletions, tables, content.startId, content.length); - return { property: content.position, value: { type: "delete-sequence" } }; - case "movable-list-insert": - return { - property: assertNonnegativePosition(content.position), - value: { type: "loro-value", value: { type: "list", value: content.values } }, - }; - case "movable-list-delete": - registerDeletion(deletions, tables, content.startId, content.length); - return { property: content.position, value: { type: "delete-sequence" } }; - case "movable-list-move": - return { - property: assertNonnegativePosition(content.to), - value: { - type: "list-move", - from: BigInt(assertNonnegativePosition(content.from)), - fromPeerIndex: BigInt(registerPeer(tables, content.elementId.peer)), - lamport: BigInt(content.elementId.lamport), - }, - }; - case "movable-list-set": - return { - property: 0, - value: { - type: "list-set", - peerIndex: BigInt(registerPeer(tables, content.elementId.peer)), - lamport: content.elementId.lamport, - value: content.value, - }, - }; - case "tree-create": - case "tree-move": - return encodeTreeMove(content.subject, content.parent, content.position, tables); - case "tree-delete": - return encodeTreeMove(content.subject, DELETED_TREE_ROOT, new Uint8Array(), tables, true); - case "future": - return { property: content.property, value: content.value }; - } -} -function encodeTreeMove(subject, parent, position, tables, deleting = false) { - return { - property: 0, - value: { - type: "raw-tree-move", - subjectPeerIndex: BigInt(registerPeer(tables, subject.peer)), - subjectCounter: subject.counter, - positionIndex: deleting ? 0n : BigInt(registerPosition(tables, position)), - parentIsNull: parent === undefined, - parentPeerIndex: parent === undefined ? 0n : BigInt(registerPeer(tables, parent.peer)), - parentCounter: parent?.counter ?? 0, - }, - }; -} -function takeDeletion(context, materialize) { - const peerIndexValue = context.deleteStartIds.peerIndices[context.deleteIndex]; - const counter = context.deleteStartIds.counters[context.deleteIndex]; - const length = context.deleteStartIds.lengths[context.deleteIndex]; - decodeAssert(peerIndexValue !== undefined && counter !== undefined && length !== undefined, "delete start ID underflow"); - context.deleteIndex += 1; - const peerIndex = bigintToIndex(peerIndexValue, context.peers.length - 1, "delete start peer index"); - return materialize - ? { - startId: { peer: context.peers[peerIndex], counter }, - length, - } - : undefined; -} -function registerDeletion(deletions, tables, startId, length) { - deletions.peerIndices.push(BigInt(registerPeer(tables, startId.peer))); - deletions.counters.push(startId.counter); - deletions.lengths.push(length); -} -function initializeTables(block) { - const peers = [...block.peers]; - const keys = [...block.keys]; - const peerIndices = new Map(); - const keyIndices = new Map(); - for (let index = 0; index < peers.length; index += 1) { - if (peerIndices.has(peers[index])) { - throw new LoroEncodeError("duplicate peer table entry"); - } - peerIndices.set(peers[index], index); - } - for (let index = 0; index < keys.length; index += 1) { - if (!keyIndices.has(keys[index])) { - keyIndices.set(keys[index], index); - } - } - const tables = { - peers, - peerIndices, - keys, - keyIndices, - containers: block.containers.map(cloneContainerId), - rootContainerIndices: new Map(), - normalContainerIndices: new Map(), - positions: block.positions.map((position) => position.slice()), - positionIndices: new Map(), - }; - // The first occurrence wins, matching the former linear-scan lookups. - for (let index = 0; index < tables.containers.length; index += 1) { - const container = tables.containers[index]; - if (lookupContainerIndex(tables, container) === undefined) { - recordContainerIndex(tables, container, index); - } - } - for (let index = 0; index < tables.positions.length; index += 1) { - const key = positionKey(tables.positions[index]); - if (!tables.positionIndices.has(key)) { - tables.positionIndices.set(key, index); - } - } - return tables; -} -function registerPeer(tables, peer) { - const current = tables.peerIndices.get(peer); - if (current !== undefined) { - return current; - } - if (peer < 0n || peer > 0xffffffffffffffffn) { - throw new LoroEncodeError(`peer ID is out of range: ${peer}`); - } - const index = tables.peers.length; - tables.peers.push(peer); - tables.peerIndices.set(peer, index); - return index; -} -function registerKey(tables, key) { - const current = tables.keyIndices.get(key); - if (current !== undefined) { - return current; - } - const index = tables.keys.length; - tables.keys.push(key); - tables.keyIndices.set(key, index); - return index; -} -function registerContainer(tables, container) { - const current = lookupContainerIndex(tables, container); - if (current !== undefined) { - return current; - } - tables.containers.push(cloneContainerId(container)); - const index = tables.containers.length - 1; - recordContainerIndex(tables, tables.containers[index], index); - return index; -} -function registerPosition(tables, position) { - const key = positionKey(position); - const current = tables.positionIndices.get(key); - if (current !== undefined) { - return current; - } - tables.positions.push(position.slice()); - const index = tables.positions.length - 1; - tables.positionIndices.set(key, index); - return index; -} -function containerTypeKey(type) { - return typeof type === "string" ? type : `#${type.value}`; -} -function lookupContainerIndex(tables, container) { - if (container.kind === "root") { - return tables.rootContainerIndices - .get(container.name) - ?.get(containerTypeKey(container.containerType)); - } - return tables.normalContainerIndices - .get(container.peer) - ?.get(container.counter) - ?.get(containerTypeKey(container.containerType)); -} -function recordContainerIndex(tables, container, index) { - const typeKey = containerTypeKey(container.containerType); - if (container.kind === "root") { - let byType = tables.rootContainerIndices.get(container.name); - if (byType === undefined) { - byType = new Map(); - tables.rootContainerIndices.set(container.name, byType); - } - byType.set(typeKey, index); - return; - } - let byCounter = tables.normalContainerIndices.get(container.peer); - if (byCounter === undefined) { - byCounter = new Map(); - tables.normalContainerIndices.set(container.peer, byCounter); - } - let byType = byCounter.get(container.counter); - if (byType === undefined) { - byType = new Map(); - byCounter.set(container.counter, byType); - } - byType.set(typeKey, index); -} -// Maps position bytes one-to-one to code units so distinct byte sequences -// never share a key. -function positionKey(position) { - let key = ""; - for (let index = 0; index < position.length; index += 1) { - key += String.fromCharCode(position[index]); - } - return key; -} -function getIndex(values, index, label) { - decodeAssert(Number.isSafeInteger(index) && index >= 0 && index < values.length, `invalid ${label} index`); - return values[index]; -} -const MAX_SAFE_BIGINT = 0x1fffffffffffffn; -function bigintToIndex(value, max, label) { - decodeAssert(value >= 0n && value <= MAX_SAFE_BIGINT, `${label} is out of range`); - const result = Number(value); - decodeAssert(max >= 0 && result <= max, `${label} is out of range`); - return result; -} -function assertNonnegativePosition(value) { - if (!Number.isSafeInteger(value) || value < 0 || value > 2147483647) { - throw new LoroEncodeError(`position is out of range: ${value}`); - } - return value; -} -function assertOperationLength(value) { - if (!Number.isSafeInteger(value) || value <= 0 || value > 2147483647) { - throw new LoroEncodeError(`operation length is out of range: ${value}`); - } -} -function checkedCounter(value) { - decodeAssert(Number.isSafeInteger(value) && value >= -2147483648 && value <= 2147483647, "operation counter overflow"); - return value; -} -function checkedEncodeCounter(value) { - if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { - throw new LoroEncodeError(`operation counter is out of range: ${value}`); - } - return value; -} -function idsEqual(left, right) { - return left.peer === right.peer && left.counter === right.counter; -} -function cloneContainerId(container) { - return container.kind === "root" - ? { - kind: "root", - name: container.name, - containerType: container.containerType, - } - : { - kind: "normal", - peer: container.peer, - counter: container.counter, - containerType: container.containerType, - }; -} diff --git a/loro-js/src/codec/change-block-tables.js b/loro-js/src/codec/change-block-tables.js deleted file mode 100644 index b7a557973..000000000 --- a/loro-js/src/codec/change-block-tables.js +++ /dev/null @@ -1,375 +0,0 @@ -import { ByteReader, ByteWriter, concatBytes } from "./bytes"; -import { containerTypeFromRawByte, containerTypeToRawByte } from "./container-id"; -import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; -import { readUlebNumber, writeUleb128 } from "./leb128"; -import { PostcardReader, PostcardWriter } from "./postcard"; -import { decodeColumnarVecMaybeWrapped, decodeDeltaRleI32, decodeDeltaRleIsize, decodeDeltaRleU32, decodeDeltaRleUsize, decodeRleU8, decodeRleU32, encodeAnyRleU32, encodeAnyRleUsize, encodeBoolRle, encodeColumnarVecWrapped, encodeDeltaOfDeltaI64, encodeDeltaRleI32, encodeDeltaRleIsize, encodeDeltaRleU32, encodeDeltaRleUsize, encodeRleU8, encodeRleU32, takeAnyRleU32, takeAnyRleUsize, takeBoolRle, takeDeltaOfDeltaI64, } from "./serde-columnar"; -const textDecoder = new TextDecoder("utf-8", { fatal: true }); -const textEncoder = new TextEncoder(); -const I32_MIN = -2147483648; -const I32_MAX = 2147483647; -export function decodeChangesHeader(bytes, options) { - const changeCount = options.changeCount; - decodeAssert(Number.isSafeInteger(changeCount) && changeCount > 0 && changeCount <= 10000000, "invalid change count"); - const firstCounter = options.counterStart | 0; - decodeAssert(options.counterLength <= I32_MAX, "change counter length is out of range"); - const reader = new ByteReader(bytes); - const peerCount = readUlebNumber(reader, 10000000); - decodeAssert(peerCount > 0, "change header has an empty peer table"); - const peers = []; - for (let index = 0; index < peerCount; index += 1) { - peers.push(reader.readU64LE()); - } - const lengths = []; - let knownLength = 0; - for (let index = 0; index < changeCount - 1; index += 1) { - const length = readUlebNumber(reader, I32_MAX); - knownLength = checkedI32(knownLength + length, "change counter length"); - lengths.push(length); - } - const finalLength = options.counterLength - knownLength; - decodeAssert(finalLength >= 0, "change lengths exceed the block counter range"); - lengths.push(finalLength); - let remaining = bytes.subarray(reader.position); - let dependencyOnSelf; - [dependencyOnSelf, remaining] = takeBoolRle(remaining, changeCount); - let dependencyLengthsBigInt; - [dependencyLengthsBigInt, remaining] = takeAnyRleUsize(remaining, changeCount); - const dependencyLengths = dependencyLengthsBigInt.map((value) => bigintToNumber(value, 10000000, "dependency length")); - const otherDependencyCount = dependencyLengths.reduce((sum, value) => sum + value, 0); - decodeAssert(otherDependencyCount <= 10000000, "too many change dependencies"); - let dependencyPeerIndices; - [dependencyPeerIndices, remaining] = takeAnyRleUsize(remaining, otherDependencyCount); - let dependencyCounters; - [dependencyCounters, remaining] = takeDeltaOfDeltaI64(remaining, otherDependencyCount); - const dependencies = []; - let dependencyIndex = 0; - let counter = firstCounter; - for (let index = 0; index < changeCount; index += 1) { - const ids = []; - if (dependencyOnSelf[index]) { - decodeAssert(counter > I32_MIN, "self dependency counter underflow"); - ids.push({ peer: peers[0], counter: counter - 1 }); - } - for (let dep = 0; dep < dependencyLengths[index]; dep += 1) { - const peerIndex = bigintToNumber(dependencyPeerIndices[dependencyIndex], peers.length - 1, "dependency peer index"); - const dependencyCounter = bigintToNumber(dependencyCounters[dependencyIndex], I32_MAX, "dependency counter"); - ids.push({ peer: peers[peerIndex], counter: dependencyCounter }); - dependencyIndex += 1; - } - dependencies.push(ids); - counter = checkedI32(counter + lengths[index], "change counter"); - } - decodeAssert(dependencyIndex === dependencyPeerIndices.length && - dependencyIndex === dependencyCounters.length, "trailing change dependencies"); - const counters = []; - counter = firstCounter; - for (const length of lengths) { - counters.push(counter); - counter = checkedI32(counter + length, "change counter"); - } - counters.push(counter); - let encodedLamports; - [encodedLamports, remaining] = takeDeltaOfDeltaI64(remaining, changeCount - 1); - decodeAssert(remaining.length === 0, "trailing change header bytes"); - const lamports = encodedLamports.map((value) => bigintToNumber(value, 4294967295, "lamport")); - const blockLamportEnd = options.lamportStart + options.lamportLength; - decodeAssert(blockLamportEnd <= 4294967295, "lamport range overflow"); - const lastLamport = blockLamportEnd - finalLength; - decodeAssert(Number.isSafeInteger(lastLamport) && lastLamport >= 0 && lastLamport <= 4294967295, "invalid final lamport"); - lamports.push(lastLamport); - return { - peer: peers[0], - peers, - counters, - lengths, - lamports, - dependencies, - }; -} -export function encodeChangesHeader(header) { - const changeCount = header.lengths.length; - if (changeCount === 0 || - header.peers.length === 0 || - header.peers[0] !== header.peer || - header.dependencies.length !== changeCount || - header.lamports.length !== changeCount) { - throw new LoroEncodeError("inconsistent change header arrays"); - } - const writer = new ByteWriter(); - writeUleb128(writer, header.peers.length); - for (const peer of header.peers) { - writer.writeU64LE(peer); - } - for (let index = 0; index < changeCount - 1; index += 1) { - writeUleb128(writer, assertNonnegativeI32(header.lengths[index], "change length")); - } - const peerIndices = new Map(header.peers.map((peer, index) => [peer, index])); - const selfDependencies = []; - const dependencyLengths = []; - const dependencyPeerIndices = []; - const dependencyCounters = []; - for (let index = 0; index < changeCount; index += 1) { - const expectedSelfCounter = header.counters[index] - 1; - let hasSelf = false; - let otherCount = 0; - for (const dependency of header.dependencies[index]) { - if (dependency.peer === header.peer) { - if (hasSelf || dependency.counter !== expectedSelfCounter) { - throw new LoroEncodeError("invalid same-peer change dependency"); - } - hasSelf = true; - } - else { - const peerIndex = peerIndices.get(dependency.peer); - if (peerIndex === undefined) { - throw new LoroEncodeError(`dependency peer is absent from the peer table`); - } - dependencyPeerIndices.push(BigInt(peerIndex)); - dependencyCounters.push(BigInt(assertNonnegativeI32(dependency.counter, "dependency counter"))); - otherCount += 1; - } - } - selfDependencies.push(hasSelf); - dependencyLengths.push(BigInt(otherCount)); - } - writer.writeBytes(encodeBoolRle(selfDependencies)); - writer.writeBytes(encodeAnyRleUsize(dependencyLengths)); - writer.writeBytes(encodeAnyRleUsize(dependencyPeerIndices)); - writer.writeBytes(encodeDeltaOfDeltaI64(dependencyCounters)); - writer.writeBytes(encodeDeltaOfDeltaI64(header.lamports.slice(0, -1).map((value) => BigInt(value)))); - return writer.toUint8Array(); -} -export function decodeChangesMetadata(bytes, changeCount) { - let remaining = bytes; - let timestamps; - [timestamps, remaining] = takeDeltaOfDeltaI64(remaining, changeCount); - let messageLengths; - [messageLengths, remaining] = takeAnyRleU32(remaining, changeCount); - const totalLength = messageLengths.reduce((sum, value) => sum + value, 0); - decodeAssert(totalLength === remaining.length, "commit message byte length mismatch"); - const commitMessages = []; - let offset = 0; - for (const length of messageLengths) { - if (length === 0) { - commitMessages.push(undefined); - continue; - } - const messageBytes = remaining.subarray(offset, offset + length); - try { - commitMessages.push(textDecoder.decode(messageBytes)); - } - catch { - throw new LoroDecodeError("invalid UTF-8 commit message", offset); - } - offset += length; - } - return { timestamps, commitMessages }; -} -export function encodeChangesMetadata(metadata) { - if (metadata.timestamps.length !== metadata.commitMessages.length) { - throw new LoroEncodeError("inconsistent change metadata arrays"); - } - const lengths = []; - const messages = []; - for (const message of metadata.commitMessages) { - if (message === undefined) { - lengths.push(0); - } - else { - const bytes = textEncoder.encode(message); - lengths.push(bytes.length); - messages.push(bytes); - } - } - return concatBytes(encodeDeltaOfDeltaI64(metadata.timestamps), encodeAnyRleU32(lengths), ...messages); -} -export function decodeChangeKeys(bytes) { - const reader = new ByteReader(bytes); - const keys = []; - while (reader.remaining > 0) { - const offset = reader.position; - const value = reader.readBytes(readUlebNumber(reader, 2147483647)); - try { - keys.push(textDecoder.decode(value)); - } - catch { - throw new LoroDecodeError("invalid UTF-8 change key", offset); - } - } - return keys; -} -export function encodeChangeKeys(keys) { - const writer = new ByteWriter(); - for (const key of keys) { - const bytes = textEncoder.encode(key); - writeUleb128(writer, bytes.length); - writer.writeBytes(bytes); - } - return writer.toUint8Array(); -} -export function decodeContainerArena(bytes, peers, keys) { - const reader = new PostcardReader(bytes); - const count = reader.readUsize(); - decodeAssert(count <= 10000000, "container arena is too large"); - const containers = []; - for (let index = 0; index < count; index += 1) { - decodeAssert(reader.readUsize() === 4, "invalid encoded container field count"); - const isRoot = reader.readBool(); - const containerType = containerTypeFromRawByte(reader.readU8()); - const peerIndex = reader.readUsize(); - const keyIndexOrCounter = reader.readI32(); - if (isRoot) { - decodeAssert(keyIndexOrCounter >= 0 && keyIndexOrCounter < keys.length, "invalid root container key index"); - containers.push({ - kind: "root", - name: keys[keyIndexOrCounter], - containerType, - }); - } - else { - decodeAssert(peerIndex < peers.length, "invalid normal container peer index"); - containers.push({ - kind: "normal", - peer: peers[peerIndex], - counter: keyIndexOrCounter, - containerType, - }); - } - } - reader.assertEnd(); - return containers; -} -export function encodeContainerArena(containers, peers, keys) { - const writer = new PostcardWriter(); - writer.writeUsize(containers.length); - const peerIndices = new Map(peers.map((peer, index) => [peer, index])); - const keyIndices = new Map(keys.map((key, index) => [key, index])); - for (const container of containers) { - writer.writeUsize(4); - writer.writeBool(container.kind === "root"); - writer.writeU8(containerTypeToRawByte(container.containerType)); - if (container.kind === "root") { - const keyIndex = keyIndices.get(container.name); - if (keyIndex === undefined) { - throw new LoroEncodeError(`root container name is absent from the key table`); - } - writer.writeUsize(0); - writer.writeI32(keyIndex); - } - else { - const peerIndex = peerIndices.get(container.peer); - if (peerIndex === undefined) { - throw new LoroEncodeError(`container peer is absent from the peer table`); - } - writer.writeUsize(peerIndex); - writer.writeI32(container.counter); - } - } - return writer.toUint8Array(); -} -export function decodeEncodedOperations(bytes) { - const decoded = decodeEncodedOperationColumns(bytes); - return decoded.containerIndices.map((containerIndex, index) => ({ - containerIndex, - property: decoded.properties[index], - valueType: decoded.valueTypes[index], - length: decoded.lengths[index], - })); -} -export function decodeEncodedOperationColumns(bytes) { - if (bytes.length === 0) { - return { containerIndices: [], properties: [], valueTypes: [], lengths: [] }; - } - const columns = decodeColumnarVecMaybeWrapped(bytes); - decodeAssert(columns.length === 4, "encoded operations must have four columns"); - const containerIndices = decodeDeltaRleU32(columns[0]); - const properties = decodeDeltaRleI32(columns[1]); - const valueTypes = decodeRleU8(columns[2]); - const lengths = decodeRleU32(columns[3]); - decodeAssert(properties.length === containerIndices.length && - valueTypes.length === containerIndices.length && - lengths.length === containerIndices.length, "encoded operation column length mismatch"); - return { containerIndices, properties, valueTypes, lengths }; -} -export function encodeEncodedOperations(rows) { - return encodeEncodedOperationColumns({ - containerIndices: rows.map((row) => row.containerIndex), - properties: rows.map((row) => row.property), - valueTypes: rows.map((row) => row.valueType), - lengths: rows.map((row) => row.length), - }); -} -/** Encodes operation columns directly, without intermediate row objects. */ -export function encodeEncodedOperationColumns(columns) { - if (columns.properties.length !== columns.containerIndices.length || - columns.valueTypes.length !== columns.containerIndices.length || - columns.lengths.length !== columns.containerIndices.length) { - throw new LoroEncodeError("inconsistent encoded operation columns"); - } - return encodeColumnarVecWrapped([ - encodeDeltaRleU32(columns.containerIndices), - encodeDeltaRleI32(columns.properties), - encodeRleU8(columns.valueTypes), - encodeRleU32(columns.lengths), - ]); -} -export function decodeDeleteStartIds(bytes) { - const decoded = decodeDeleteStartIdColumns(bytes); - return decoded.peerIndices.map((peerIndex, index) => ({ - peerIndex, - counter: decoded.counters[index], - length: decoded.lengths[index], - })); -} -export function decodeDeleteStartIdColumns(bytes) { - if (bytes.length === 0) { - return { peerIndices: [], counters: [], lengths: [] }; - } - const columns = decodeColumnarVecMaybeWrapped(bytes); - decodeAssert(columns.length === 3, "delete start IDs must have three columns"); - const peerIndices = decodeDeltaRleUsize(columns[0]); - const counters = decodeDeltaRleI32(columns[1]); - const lengths = decodeDeltaRleIsize(columns[2]); - decodeAssert(counters.length === peerIndices.length && lengths.length === peerIndices.length, "delete start ID column length mismatch"); - return { peerIndices, counters, lengths }; -} -export function encodeDeleteStartIds(rows) { - return encodeDeleteStartIdColumns({ - peerIndices: rows.map((row) => row.peerIndex), - counters: rows.map((row) => row.counter), - lengths: rows.map((row) => row.length), - }); -} -/** Encodes delete start ID columns directly, without intermediate row objects. */ -export function encodeDeleteStartIdColumns(columns) { - if (columns.counters.length !== columns.peerIndices.length || - columns.lengths.length !== columns.peerIndices.length) { - throw new LoroEncodeError("inconsistent delete start ID columns"); - } - if (columns.peerIndices.length === 0) { - return new Uint8Array(); - } - return encodeColumnarVecWrapped([ - encodeDeltaRleUsize(columns.peerIndices), - encodeDeltaRleI32(columns.counters), - encodeDeltaRleIsize(columns.lengths), - ]); -} -function checkedI32(value, label) { - decodeAssert(Number.isSafeInteger(value) && value >= I32_MIN && value <= I32_MAX, `${label} overflow`); - return value; -} -const MAX_SAFE_BIGINT = 0x1fffffffffffffn; -function bigintToNumber(value, max, label) { - decodeAssert(value >= 0n && value <= MAX_SAFE_BIGINT, `${label} is out of range`); - const result = Number(value); - decodeAssert(result <= max, `${label} is out of range`); - return result; -} -function assertNonnegativeI32(value, label) { - if (!Number.isSafeInteger(value) || value < 0 || value > I32_MAX) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } - return value; -} diff --git a/loro-js/src/codec/change-block.js b/loro-js/src/codec/change-block.js deleted file mode 100644 index e7ddc2658..000000000 --- a/loro-js/src/codec/change-block.js +++ /dev/null @@ -1,38 +0,0 @@ -import { PostcardReader, PostcardWriter } from "./postcard"; -export function decodeEncodedChangeBlock(bytes) { - const reader = new PostcardReader(bytes); - const block = { - counterStart: reader.readU32(), - counterLength: reader.readU32(), - lamportStart: reader.readU32(), - lamportLength: reader.readU32(), - changeCount: reader.readU32(), - header: reader.readBytes(), - changeMetadata: reader.readBytes(), - containerIds: reader.readBytes(), - keys: reader.readBytes(), - positions: reader.readBytes(), - operations: reader.readBytes(), - deleteStartIds: reader.readBytes(), - values: reader.readBytes(), - }; - reader.assertEnd(); - return block; -} -export function encodeEncodedChangeBlock(block) { - const writer = new PostcardWriter(); - writer.writeU32(block.counterStart); - writer.writeU32(block.counterLength); - writer.writeU32(block.lamportStart); - writer.writeU32(block.lamportLength); - writer.writeU32(block.changeCount); - writer.writeBytes(block.header); - writer.writeBytes(block.changeMetadata); - writer.writeBytes(block.containerIds); - writer.writeBytes(block.keys); - writer.writeBytes(block.positions); - writer.writeBytes(block.operations); - writer.writeBytes(block.deleteStartIds); - writer.writeBytes(block.values); - return writer.toUint8Array(); -} diff --git a/loro-js/src/codec/change-value.js b/loro-js/src/codec/change-value.js deleted file mode 100644 index 58c2c537f..000000000 --- a/loro-js/src/codec/change-value.js +++ /dev/null @@ -1,375 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; -import { readSleb128, readUleb128, writeSleb128, writeUleb128 } from "./leb128"; -const textDecoder = new TextDecoder("utf-8", { fatal: true }); -const textEncoder = new TextEncoder(); -const MAX_VALUE_DEPTH = 1024; -const MAX_COLLECTION_LENGTH = 10000000; -const MAX_COLLECTION_LENGTH_BIGINT = BigInt(MAX_COLLECTION_LENGTH); -const I32_MIN_BIGINT = -0x80000000n; -const I32_MAX_BIGINT = 0x7fffffffn; -const U32_MAX_BIGINT = 0xffffffffn; -// Shared scratch for big-endian f64 access; never exposed to callers. -const f64Scratch = new Uint8Array(8); -const f64View = new DataView(f64Scratch.buffer); -export function decodeChangeValue(bytes) { - const reader = new ByteReader(bytes); - const tag = reader.readU8(); - const value = readChangeValueContent(tag, reader); - decodeAssert(reader.remaining === 0, "trailing change value bytes"); - return value; -} -export function decodeChangeValueContent(tag, bytes) { - const reader = new ByteReader(bytes); - const value = readChangeValueContent(tag, reader); - return [value, bytes.subarray(reader.position)]; -} -/** - * Reads one tagged change value payload from the reader, leaving it positioned - * after the value. Per-operation decode loops share one reader across values to - * avoid the tuple and subarray allocations of decodeChangeValueContent. - */ -export function readChangeValueContent(tag, reader) { - const kind = tag & 0x7f; - switch (kind) { - case 0: - return { type: "null" }; - case 1: - return { type: "bool", value: true }; - case 2: - return { type: "bool", value: false }; - case 3: - return { type: "i64", value: readSleb128(reader) }; - case 4: - return { type: "double", value: readF64BE(reader) }; - case 5: - return { type: "string", value: readUtf8(reader) }; - case 6: - return { type: "binary", value: readLengthPrefixedBytes(reader) }; - case 7: - return { type: "container-index", value: readUleb128(reader) }; - case 8: - return { type: "delete-once" }; - case 9: - return { type: "delete-sequence" }; - case 10: { - const integer = readSleb128(reader); - decodeAssert(integer >= I32_MIN_BIGINT && integer <= I32_MAX_BIGINT, "change delta integer is out of range"); - return { type: "delta-int", value: Number(integer) }; - } - case 11: - return { type: "loro-value", value: readChangeLoroValue(reader) }; - case 12: - return { - type: "mark-start", - info: reader.readU8(), - length: readUleb128(reader), - keyIndex: readUleb128(reader), - value: readChangeLoroValue(reader), - }; - case 13: { - const targetIndex = readUleb128(reader); - const parentIsNull = reader.readU8() !== 0; - const position = readUleb128(reader); - return { - type: "tree-move", - targetIndex, - parentIsNull, - position, - parentIndex: parentIsNull ? undefined : readUleb128(reader), - }; - } - case 14: - return { - type: "list-move", - from: readUleb128(reader), - fromPeerIndex: readUleb128(reader), - lamport: readUleb128(reader), - }; - case 15: { - const peerIndex = readUleb128(reader); - const lamport = readUleb128(reader); - decodeAssert(lamport <= U32_MAX_BIGINT, "list-set lamport is out of range"); - return { - type: "list-set", - peerIndex, - lamport: Number(lamport), - value: readChangeLoroValue(reader), - }; - } - case 16: { - const subjectPeerIndex = readUleb128(reader); - const subjectCounter = readNonnegativeI32(reader, "tree subject counter"); - const positionIndex = readUleb128(reader); - const parentIsNull = reader.readU8() !== 0; - const parentPeerIndex = parentIsNull ? 0n : readUleb128(reader); - const parentCounter = parentIsNull - ? 0 - : readNonnegativeI32(reader, "tree parent counter"); - return { - type: "raw-tree-move", - subjectPeerIndex, - subjectCounter, - positionIndex, - parentIsNull, - parentPeerIndex, - parentCounter, - }; - } - default: - return { type: "future", tag, data: readLengthPrefixedBytes(reader) }; - } -} -export function encodeChangeValue(value) { - const encoded = encodeChangeValueContent(value); - const writer = new ByteWriter(encoded.bytes.length + 1); - writer.writeU8(encoded.tag); - writer.writeBytes(encoded.bytes); - return writer.toUint8Array(); -} -export function encodeChangeValueContent(value) { - const writer = new ByteWriter(); - let tag; - switch (value.type) { - case "null": - tag = 0; - break; - case "bool": - tag = value.value ? 1 : 2; - break; - case "i64": - tag = 3; - writeSleb128(writer, value.value); - break; - case "double": - tag = 4; - writeF64BE(writer, value.value); - break; - case "string": - tag = 5; - writeUtf8(writer, value.value); - break; - case "binary": - tag = 6; - writeLengthPrefixedBytes(writer, value.value); - break; - case "container-index": - tag = 7; - writeUleb128(writer, value.value); - break; - case "delete-once": - tag = 8; - break; - case "delete-sequence": - tag = 9; - break; - case "delta-int": - tag = 10; - assertI32(value.value, "delta integer"); - writeSleb128(writer, value.value); - break; - case "loro-value": - tag = 11; - writeChangeLoroValue(writer, value.value); - break; - case "mark-start": - tag = 12; - writer.writeU8(value.info); - writeUleb128(writer, value.length); - writeUleb128(writer, value.keyIndex); - writeChangeLoroValue(writer, value.value); - break; - case "tree-move": - tag = 13; - writeUleb128(writer, value.targetIndex); - writer.writeU8(value.parentIsNull ? 1 : 0); - writeUleb128(writer, value.position); - if (!value.parentIsNull) { - if (value.parentIndex === undefined) { - throw new LoroEncodeError("tree move lacks its parent index"); - } - writeUleb128(writer, value.parentIndex); - } - break; - case "list-move": - tag = 14; - writeUleb128(writer, value.from); - writeUleb128(writer, value.fromPeerIndex); - writeUleb128(writer, value.lamport); - break; - case "list-set": - tag = 15; - writeUleb128(writer, value.peerIndex); - writeUleb128(writer, value.lamport); - writeChangeLoroValue(writer, value.value); - break; - case "raw-tree-move": - tag = 16; - writeUleb128(writer, value.subjectPeerIndex); - writeUleb128(writer, assertNonnegativeI32(value.subjectCounter, "tree subject counter")); - writeUleb128(writer, value.positionIndex); - writer.writeU8(value.parentIsNull ? 1 : 0); - if (!value.parentIsNull) { - writeUleb128(writer, value.parentPeerIndex); - writeUleb128(writer, assertNonnegativeI32(value.parentCounter, "tree parent counter")); - } - break; - case "future": - tag = value.tag; - if (!Number.isSafeInteger(tag) || tag < 0 || tag > 0xff || (tag & 0x7f) <= 16) { - throw new LoroEncodeError(`invalid future change value tag: ${tag}`); - } - writeLengthPrefixedBytes(writer, value.data); - } - return { tag, bytes: writer.toUint8Array() }; -} -export function readChangeLoroValue(reader, depth = 0) { - decodeAssert(depth <= MAX_VALUE_DEPTH, "change LoroValue is too deep", reader.position); - const kind = reader.readU8(); - switch (kind) { - case 0: - return { type: "null" }; - case 1: - return { type: "bool", value: true }; - case 2: - return { type: "bool", value: false }; - case 3: - return { type: "i64", value: readSleb128(reader) }; - case 4: - return { type: "double", value: readF64BE(reader) }; - case 5: - return { type: "string", value: readUtf8(reader) }; - case 6: - return { type: "binary", value: readLengthPrefixedBytes(reader) }; - case 7: { - const length = readCollectionLength(reader); - decodeAssert(length <= reader.remaining, "change LoroValue list length exceeds input"); - const value = []; - for (let index = 0; index < length; index += 1) { - value.push(readChangeLoroValue(reader, depth + 1)); - } - return { type: "list", value }; - } - case 8: { - const length = readCollectionLength(reader); - decodeAssert(length <= Math.floor(reader.remaining / 2), "change LoroValue map length exceeds input"); - const value = []; - for (let index = 0; index < length; index += 1) { - value.push([readUleb128(reader), readChangeLoroValue(reader, depth + 1)]); - } - return { type: "map", value }; - } - case 9: - return { type: "container-type", value: reader.readU8() }; - default: - throw new LoroDecodeError("invalid change LoroValue kind", reader.position - 1); - } -} -export function writeChangeLoroValue(writer, value, depth = 0) { - if (depth > MAX_VALUE_DEPTH) { - throw new LoroEncodeError("change LoroValue is too deep"); - } - switch (value.type) { - case "null": - writer.writeU8(0); - return; - case "bool": - writer.writeU8(value.value ? 1 : 2); - return; - case "i64": - writer.writeU8(3); - writeSleb128(writer, value.value); - return; - case "double": - writer.writeU8(4); - writeF64BE(writer, value.value); - return; - case "string": - writer.writeU8(5); - writeUtf8(writer, value.value); - return; - case "binary": - writer.writeU8(6); - writeLengthPrefixedBytes(writer, value.value); - return; - case "list": - writer.writeU8(7); - assertCollectionLength(value.value.length); - writeUleb128(writer, value.value.length); - for (const item of value.value) { - writeChangeLoroValue(writer, item, depth + 1); - } - return; - case "map": - writer.writeU8(8); - assertCollectionLength(value.value.length); - writeUleb128(writer, value.value.length); - for (const [keyIndex, item] of value.value) { - writeUleb128(writer, keyIndex); - writeChangeLoroValue(writer, item, depth + 1); - } - return; - case "container-type": - writer.writeU8(9); - writer.writeU8(value.value); - } -} -function readCollectionLength(reader) { - const value = readUleb128(reader, MAX_COLLECTION_LENGTH_BIGINT); - return Number(value); -} -function readNonnegativeI32(reader, label) { - const value = readUleb128(reader, I32_MAX_BIGINT); - decodeAssert(value <= I32_MAX_BIGINT, `${label} is out of range`); - return Number(value); -} -function readUtf8(reader) { - const offset = reader.position; - const bytes = readLengthPrefixedBytes(reader); - try { - return textDecoder.decode(bytes); - } - catch { - throw new LoroDecodeError("invalid UTF-8 change value", offset); - } -} -function writeUtf8(writer, value) { - writeLengthPrefixedBytes(writer, textEncoder.encode(value)); -} -function readLengthPrefixedBytes(reader) { - const length = readUleb128(reader, I32_MAX_BIGINT); - return reader.readBytes(Number(length)); -} -function writeLengthPrefixedBytes(writer, value) { - writeUleb128(writer, value.length); - writer.writeBytes(value); -} -function readF64BE(reader) { - const offset = reader.position; - reader.skip(8); - const bytes = reader.bytes; - for (let index = 0; index < 8; index += 1) { - f64Scratch[index] = bytes[offset + index]; - } - return f64View.getFloat64(0, false); -} -function writeF64BE(writer, value) { - f64View.setFloat64(0, value, false); - writer.writeBytes(f64Scratch); -} -function assertI32(value, label) { - if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } -} -function assertNonnegativeI32(value, label) { - if (!Number.isSafeInteger(value) || value < 0 || value > 2147483647) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } - return value; -} -function assertCollectionLength(length) { - if (!Number.isSafeInteger(length) || length < 0 || length > MAX_COLLECTION_LENGTH) { - throw new LoroEncodeError(`change value collection is too large: ${length}`); - } -} diff --git a/loro-js/src/codec/container-id.js b/loro-js/src/codec/container-id.js deleted file mode 100644 index b22800cfb..000000000 --- a/loro-js/src/codec/container-id.js +++ /dev/null @@ -1,225 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroDecodeError, LoroEncodeError } from "./errors"; -import { readUlebNumber, writeUleb128 } from "./leb128"; -import { PostcardReader, PostcardWriter } from "./postcard"; -import { ContainerType, } from "./types"; -const textDecoder = new TextDecoder("utf-8", { fatal: true }); -const textEncoder = new TextEncoder(); -export function unknownContainerType(value) { - assertU8(value, "unknown container type"); - return { kind: "unknown", value }; -} -export function containerTypeToRawByte(type) { - switch (type) { - case ContainerType.Map: - return 0; - case ContainerType.List: - return 1; - case ContainerType.Text: - return 2; - case ContainerType.Tree: - return 3; - case ContainerType.MovableList: - return 4; - case ContainerType.Counter: - return 5; - default: - assertU8(type.value, "unknown container type"); - return type.value; - } -} -export function containerTypeFromRawByte(value) { - switch (value) { - case 0: - return ContainerType.Map; - case 1: - return ContainerType.List; - case 2: - return ContainerType.Text; - case 3: - return ContainerType.Tree; - case 4: - return ContainerType.MovableList; - case 5: - return ContainerType.Counter; - default: - return { kind: "unknown", value }; - } -} -export function containerTypeToHistoricalByte(type) { - switch (type) { - case ContainerType.Text: - return 0; - case ContainerType.Map: - return 1; - case ContainerType.List: - return 2; - case ContainerType.MovableList: - return 3; - case ContainerType.Tree: - return 4; - case ContainerType.Counter: - return 5; - default: - assertU8(type.value, "unknown container type"); - return type.value; - } -} -export function containerTypeFromHistoricalByte(value) { - switch (value) { - case 0: - return ContainerType.Text; - case 1: - return ContainerType.Map; - case 2: - return ContainerType.List; - case 3: - return ContainerType.MovableList; - case 4: - return ContainerType.Tree; - case 5: - return ContainerType.Counter; - default: - return { kind: "unknown", value }; - } -} -export function decodeContainerId(bytes) { - if (bytes.length === 0) { - throw new LoroDecodeError("container ID is empty"); - } - const reader = new ByteReader(bytes); - const first = reader.readU8(); - const containerType = containerTypeFromRawByte(first & 0x7f); - if ((first & 0x80) !== 0) { - const nameOffset = reader.position; - const length = readUlebNumber(reader, 2147483647); - let name; - try { - name = textDecoder.decode(reader.readBytes(length)); - } - catch (error) { - if (error instanceof LoroDecodeError) { - throw error; - } - throw new LoroDecodeError("invalid UTF-8 container name", nameOffset); - } - reader.assertEnd("trailing root container ID bytes"); - return { kind: "root", name, containerType }; - } - if (reader.remaining !== 12) { - throw new LoroDecodeError("invalid normal container ID length", reader.position); - } - const peer = reader.readU64LE(); - const counter = reader.readU32LE() | 0; - reader.assertEnd("trailing normal container ID bytes"); - return { kind: "normal", peer, counter, containerType }; -} -export function encodeContainerId(id) { - const rawType = containerTypeToRawByte(id.containerType); - if (rawType > 0x7f) { - throw new LoroEncodeError(`raw container type is out of range: ${rawType}`); - } - const writer = new ByteWriter(); - if (id.kind === "root") { - const name = textEncoder.encode(id.name); - writer.writeU8(rawType | 0x80); - writeUleb128(writer, name.length); - writer.writeBytes(name); - return writer.toUint8Array(); - } - assertNormalContainerId(id); - writer.writeU8(rawType); - writer.writeU64LE(id.peer); - writer.writeU32LE(id.counter >>> 0); - return writer.toUint8Array(); -} -export function readPostcardContainerId(reader) { - const tag = reader.readU32(); - if (tag === 0) { - return { - kind: "root", - name: reader.readString(), - containerType: containerTypeFromHistoricalByte(reader.readU8()), - }; - } - if (tag === 1) { - return { - kind: "normal", - peer: reader.readU64(), - counter: reader.readI32(), - containerType: containerTypeFromHistoricalByte(reader.readU8()), - }; - } - throw new LoroDecodeError("invalid postcard ContainerID tag", reader.input.position); -} -export function writePostcardContainerId(writer, id) { - if (id.kind === "root") { - writer.writeU32(0); - writer.writeString(id.name); - } - else { - assertNormalContainerId(id); - writer.writeU32(1); - writer.writeU64(id.peer); - writer.writeI32(id.counter); - } - writer.writeU8(containerTypeToHistoricalByte(id.containerType)); -} -export function decodePostcardContainerId(bytes) { - const reader = new PostcardReader(bytes); - const id = readPostcardContainerId(reader); - reader.assertEnd(); - return id; -} -export function encodePostcardContainerId(id) { - const writer = new PostcardWriter(); - writePostcardContainerId(writer, id); - return writer.toUint8Array(); -} -export function readPostcardOptionalContainerId(reader) { - const tag = reader.readU32(); - if (tag === 0) { - return undefined; - } - if (tag === 1) { - return readPostcardContainerId(reader); - } - throw new LoroDecodeError("invalid postcard Option tag", reader.input.position); -} -export function writePostcardOptionalContainerId(writer, id) { - if (id === undefined) { - writer.writeU32(0); - return; - } - writer.writeU32(1); - writePostcardContainerId(writer, id); -} -export function decodePostcardOptionalContainerId(bytes) { - const reader = new PostcardReader(bytes); - const id = readPostcardOptionalContainerId(reader); - reader.assertEnd(); - return id; -} -export function encodePostcardOptionalContainerId(id) { - const writer = new PostcardWriter(); - writePostcardOptionalContainerId(writer, id); - return writer.toUint8Array(); -} -function assertNormalContainerId(id) { - if (id.peer < 0n || id.peer > 0xffffffffffffffffn) { - throw new LoroEncodeError(`container peer ID is out of range: ${id.peer}`); - } - if (!Number.isSafeInteger(id.counter) || - id.counter < -2147483648 || - id.counter > 2147483647) { - throw new LoroEncodeError(`container counter is out of range: ${id.counter}`); - } -} -function assertU8(value, name) { - if (!Number.isSafeInteger(value) || value < 0 || value > 0xff) { - throw new LoroEncodeError(`${name} is out of range: ${value}`); - } -} -export function isKnownContainerType(type) { - return typeof type === "string"; -} diff --git a/loro-js/src/codec/document.js b/loro-js/src/codec/document.js deleted file mode 100644 index 41d3d7a53..000000000 --- a/loro-js/src/codec/document.js +++ /dev/null @@ -1,115 +0,0 @@ -import { ByteReader, ByteWriter, bytesEqual } from "./bytes"; -import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; -import { readUlebNumber, writeUleb128 } from "./leb128"; -import { LORO_XXHASH_SEED, xxhash32 } from "./xxhash32"; -const DOCUMENT_MAGIC = Uint8Array.of(0x6c, 0x6f, 0x72, 0x6f); -const DOCUMENT_HEADER_LENGTH = 22; -export var EncodeMode; -(function (EncodeMode) { - EncodeMode[EncodeMode["FastSnapshot"] = 3] = "FastSnapshot"; - EncodeMode[EncodeMode["FastUpdates"] = 4] = "FastUpdates"; -})(EncodeMode || (EncodeMode = {})); -export function decodeDocument(bytes, options = {}) { - const reader = new ByteReader(bytes); - decodeAssert(bytes.length >= DOCUMENT_HEADER_LENGTH, "document is too short", 0); - decodeAssert(bytesEqual(reader.readBytes(4), DOCUMENT_MAGIC), "invalid document magic", 0); - const checksumPrefix = reader.readBytes(12); - if (options.requireCanonicalPrefix === true) { - for (const byte of checksumPrefix) { - decodeAssert(byte === 0, "nonzero document checksum prefix", 4); - } - } - const checksum = reader.readU32LE(); - const rawMode = reader.readU16BE(); - if (rawMode !== EncodeMode.FastSnapshot && rawMode !== EncodeMode.FastUpdates) { - throw new LoroDecodeError(`unsupported document mode ${rawMode}`, 20); - } - const body = reader.readRemaining(); - if (options.checkChecksum !== false) { - const expected = xxhash32(bytes.subarray(20), LORO_XXHASH_SEED); - decodeAssert(checksum === expected, "document checksum mismatch", 16); - } - return { mode: rawMode, body, checksum, checksumPrefix }; -} -export function encodeDocument(mode, body) { - if (mode !== EncodeMode.FastSnapshot && mode !== EncodeMode.FastUpdates) { - throw new LoroEncodeError(`unsupported document mode ${mode}`); - } - // Header and body share one allocation so the body is copied exactly once; - // the checksum covers the mode and body bytes and is backfilled afterwards. - const output = new Uint8Array(DOCUMENT_HEADER_LENGTH + body.length); - output.set(DOCUMENT_MAGIC, 0); - // Bytes 4..16 stay zero as the checksum prefix. - output[20] = mode >>> 8; - output[21] = mode & 0xff; - output.set(body, DOCUMENT_HEADER_LENGTH); - const checksum = xxhash32(output.subarray(20), LORO_XXHASH_SEED); - output[16] = checksum & 0xff; - output[17] = (checksum >>> 8) & 0xff; - output[18] = (checksum >>> 16) & 0xff; - output[19] = (checksum >>> 24) & 0xff; - return output; -} -export function decodeFastSnapshotBody(body) { - const reader = new ByteReader(body); - const oplog = readU32LengthPrefixed(reader, "oplog"); - const state = readU32LengthPrefixed(reader, "state"); - const shallowRootState = readU32LengthPrefixed(reader, "shallow root state"); - reader.assertEnd("trailing FastSnapshot bytes"); - return { oplog, state, shallowRootState }; -} -export function encodeFastSnapshotBody(snapshot) { - const writer = new ByteWriter(12 + snapshot.oplog.length + snapshot.state.length + snapshot.shallowRootState.length); - writeU32LengthPrefixed(writer, snapshot.oplog, "oplog"); - writeU32LengthPrefixed(writer, snapshot.state, "state"); - writeU32LengthPrefixed(writer, snapshot.shallowRootState, "shallow root state"); - return writer.toUint8Array(); -} -export function decodeFastUpdatesBody(body) { - const reader = new ByteReader(body); - const blocks = []; - while (reader.remaining > 0) { - const length = readUlebNumber(reader, 4294967295); - blocks.push(reader.readBytes(length)); - } - return blocks; -} -export function encodeFastUpdatesBody(blocks) { - const writer = new ByteWriter(); - for (const block of blocks) { - writeUleb128(writer, block.length); - writer.writeBytes(block); - } - return writer.toUint8Array(); -} -export function decodeFastSnapshot(bytes, options) { - const document = decodeDocument(bytes, options); - decodeAssert(document.mode === EncodeMode.FastSnapshot, "document is not a FastSnapshot", 20); - return decodeFastSnapshotBody(document.body); -} -export function encodeFastSnapshot(snapshot) { - return encodeDocument(EncodeMode.FastSnapshot, encodeFastSnapshotBody(snapshot)); -} -export function decodeFastUpdates(bytes, options) { - const document = decodeDocument(bytes, options); - decodeAssert(document.mode === EncodeMode.FastUpdates, "document is not FastUpdates", 20); - return decodeFastUpdatesBody(document.body); -} -export function encodeFastUpdates(blocks) { - return encodeDocument(EncodeMode.FastUpdates, encodeFastUpdatesBody(blocks)); -} -function readU32LengthPrefixed(reader, label) { - const offset = reader.position; - const length = reader.readU32LE(); - if (length > reader.remaining) { - throw new LoroDecodeError(`${label} length exceeds remaining input`, offset); - } - return reader.readBytes(length); -} -function writeU32LengthPrefixed(writer, value, label) { - if (value.length > 4294967295) { - throw new LoroEncodeError(`${label} is too large`); - } - writer.writeU32LE(value.length); - writer.writeBytes(value); -} diff --git a/loro-js/src/codec/errors.js b/loro-js/src/codec/errors.js deleted file mode 100644 index 447b2d98f..000000000 --- a/loro-js/src/codec/errors.js +++ /dev/null @@ -1,24 +0,0 @@ -export class LoroDecodeError extends Error { - offset; - constructor(message, offset) { - super(offset === undefined ? message : `${message} at byte ${offset}`); - this.name = "LoroDecodeError"; - this.offset = offset; - } -} -export class LoroEncodeError extends Error { - constructor(message) { - super(message); - this.name = "LoroEncodeError"; - } -} -export function decodeAssert(condition, message, offset) { - if (!condition) { - throw new LoroDecodeError(message, offset); - } -} -export function encodeAssert(condition, message) { - if (!condition) { - throw new LoroEncodeError(message); - } -} diff --git a/loro-js/src/codec/id.js b/loro-js/src/codec/id.js deleted file mode 100644 index 7682d7abc..000000000 --- a/loro-js/src/codec/id.js +++ /dev/null @@ -1,53 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroDecodeError, LoroEncodeError } from "./errors"; -import { PostcardReader, PostcardWriter } from "./postcard"; -const I32_MIN = -2147483648; -const I32_MAX = 2147483647; -const U64_MAX = 0xffffffffffffffffn; -export function decodeChangeBlockKey(bytes) { - if (bytes.length !== 12) { - throw new LoroDecodeError("invalid change block key length"); - } - const reader = new ByteReader(bytes); - const peer = reader.readU64BE(); - const counter = reader.readU32BE() | 0; - reader.assertEnd("trailing change block key bytes"); - return { peer, counter }; -} -export function encodeChangeBlockKey(id) { - assertId(id); - const writer = new ByteWriter(12); - writer.writeU64BE(id.peer); - writer.writeU32BE(id.counter >>> 0); - return writer.toUint8Array(); -} -export function readPostcardId(reader) { - return { - peer: reader.readU64(), - counter: reader.readI32(), - }; -} -export function writePostcardId(writer, id) { - assertId(id); - writer.writeU64(id.peer); - writer.writeI32(id.counter); -} -export function decodePostcardId(bytes) { - const reader = new PostcardReader(bytes); - const id = readPostcardId(reader); - reader.assertEnd(); - return id; -} -export function encodePostcardId(id) { - const writer = new PostcardWriter(); - writePostcardId(writer, id); - return writer.toUint8Array(); -} -export function assertId(id) { - if (id.peer < 0n || id.peer > U64_MAX) { - throw new LoroEncodeError(`peer ID is out of range: ${id.peer}`); - } - if (!Number.isSafeInteger(id.counter) || id.counter < I32_MIN || id.counter > I32_MAX) { - throw new LoroEncodeError(`ID counter is out of range: ${id.counter}`); - } -} diff --git a/loro-js/src/codec/index.js b/loro-js/src/codec/index.js deleted file mode 100644 index a9a320222..000000000 --- a/loro-js/src/codec/index.js +++ /dev/null @@ -1,20 +0,0 @@ -export * from "./bytes"; -export * from "./change-block"; -export * from "./change-block-codec"; -export * from "./change-block-tables"; -export * from "./change-value"; -export * from "./container-id"; -export * from "./document"; -export * from "./errors"; -export * from "./id"; -export * from "./leb128"; -export * from "./lz4"; -export * from "./postcard"; -export * from "./position-arena"; -export { decodeAnyRleI32, decodeAnyRleU32, decodeAnyRleU64, decodeAnyRleUsize, decodeBoolRle, decodeColumnarVec, decodeColumnarVecMaybeWrapped, decodeDeltaOfDeltaI64, decodeDeltaRleI32, decodeDeltaRleIsize, decodeDeltaRleU32, decodeDeltaRleUsize, decodeRleU8, decodeRleU32, encodeAnyRleI32, encodeAnyRleU32, encodeAnyRleU64, encodeAnyRleUsize, encodeBoolRle, encodeColumnarVec, encodeColumnarVecWrapped, encodeDeltaOfDeltaI64, encodeDeltaRleI32, encodeDeltaRleIsize, encodeDeltaRleU32, encodeDeltaRleUsize, encodeRleU8, encodeRleU32, takeAnyRleI32, takeAnyRleU32, takeAnyRleUsize, takeBoolRle, takeColumnarVec, takeDeltaOfDeltaI64, } from "./serde-columnar"; -export * from "./sstable"; -export * from "./state-snapshot"; -export * from "./types"; -export * from "./value"; -export * from "./version"; -export * from "./xxhash32"; diff --git a/loro-js/src/codec/leb128.js b/loro-js/src/codec/leb128.js deleted file mode 100644 index 1076723c4..000000000 --- a/loro-js/src/codec/leb128.js +++ /dev/null @@ -1,149 +0,0 @@ -import { LoroDecodeError, LoroEncodeError } from "./errors"; -export const U64_MAX = 0xffffffffffffffffn; -export const I64_MIN = -0x8000000000000000n; -export const I64_MAX = 0x7fffffffffffffffn; -export function readUleb128(reader, max = U64_MAX) { - let result = 0n; - let shift = 0n; - const start = reader.position; - for (let index = 0; index < 10; index += 1) { - const byte = reader.readU8(); - const payload = BigInt(byte & 0x7f); - if (shift === 63n && payload > 1n) { - throw new LoroDecodeError("ULEB128 overflow", start); - } - result |= payload << shift; - if ((byte & 0x80) === 0) { - if (result > max) { - throw new LoroDecodeError("ULEB128 value is out of range", start); - } - return result; - } - shift += 7n; - } - throw new LoroDecodeError("ULEB128 overflow", start); -} -export function readUlebNumber(reader, max = Number.MAX_SAFE_INTEGER) { - if (!Number.isSafeInteger(max) || max < 0) { - throw new LoroDecodeError("invalid ULEB128 number limit", reader.position); - } - const start = reader.position; - let result = 0; - let multiplier = 1; - // Set once the value provably exceeds Number.MAX_SAFE_INTEGER (and therefore - // any max). Bytes are still consumed with readUleb128's overflow rules so - // the reader position and error behavior match the BigInt implementation. - let exceedsSafeInteger = false; - for (let index = 0; index < 10; index += 1) { - const byte = reader.readU8(); - const payload = byte & 0x7f; - if (index === 9 && payload > 1) { - throw new LoroDecodeError("ULEB128 overflow", start); - } - if (!exceedsSafeInteger && payload !== 0) { - if (multiplier > Number.MAX_SAFE_INTEGER / payload) { - exceedsSafeInteger = true; - } - else { - const product = payload * multiplier; - if (result > Number.MAX_SAFE_INTEGER - product) { - exceedsSafeInteger = true; - } - else { - result += product; - } - } - } - if ((byte & 0x80) === 0) { - if (exceedsSafeInteger || result > max) { - throw new LoroDecodeError("ULEB128 value is out of range", start); - } - return result; - } - multiplier *= 128; - } - throw new LoroDecodeError("ULEB128 overflow", start); -} -export function writeUleb128(writer, input) { - if (typeof input === "number" && - Number.isSafeInteger(input) && - input >= 0 && - input < 2147483648) { - // Smi fast path: encode small numbers without any BigInt arithmetic. - let value = input; - do { - let byte = value & 0x7f; - value >>>= 7; - if (value !== 0) { - byte |= 0x80; - } - writer.writeU8(byte); - } while (value !== 0); - return; - } - let value = typeof input === "number" ? numberToBigInt(input, "ULEB128") : input; - if (value < 0n || value > U64_MAX) { - throw new LoroEncodeError(`ULEB128 value is out of range: ${value}`); - } - do { - let byte = Number(value & 0x7fn); - value >>= 7n; - if (value !== 0n) { - byte |= 0x80; - } - writer.writeU8(byte); - } while (value !== 0n); -} -export function readSleb128(reader) { - let result = 0n; - let shift = 0n; - let byte = 0; - const start = reader.position; - for (let index = 0; index < 10; index += 1) { - byte = reader.readU8(); - const payload = BigInt(byte & 0x7f); - result |= payload << shift; - shift += 7n; - if ((byte & 0x80) === 0) { - if (index === 9) { - if (byte !== 0 && byte !== 0x7f) { - throw new LoroDecodeError("SLEB128 overflow", start); - } - return BigInt.asIntN(64, result); - } - if ((byte & 0x40) !== 0) { - result |= -1n << shift; - } - if (result < I64_MIN || result > I64_MAX) { - throw new LoroDecodeError("SLEB128 value is out of range", start); - } - return result; - } - } - throw new LoroDecodeError("SLEB128 overflow", start); -} -export function writeSleb128(writer, input) { - let value = typeof input === "number" ? numberToBigInt(input, "SLEB128") : input; - if (value < I64_MIN || value > I64_MAX) { - throw new LoroEncodeError(`SLEB128 value is out of range: ${value}`); - } - for (;;) { - let byte = Number(value & 0x7fn); - value >>= 7n; - const sign = (byte & 0x40) !== 0; - const done = (value === 0n && !sign) || (value === -1n && sign); - if (!done) { - byte |= 0x80; - } - writer.writeU8(byte); - if (done) { - return; - } - } -} -function numberToBigInt(value, name) { - if (!Number.isSafeInteger(value)) { - throw new LoroEncodeError(`${name} number must be a safe integer: ${value}`); - } - return BigInt(value); -} diff --git a/loro-js/src/codec/lz4.js b/loro-js/src/codec/lz4.js deleted file mode 100644 index bdf4f6761..000000000 --- a/loro-js/src/codec/lz4.js +++ /dev/null @@ -1,307 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroDecodeError, LoroEncodeError, decodeAssert } from "./errors"; -import { xxhash32 } from "./xxhash32"; -const LZ4_MAGIC = 407708164; -const WINDOW_SIZE = 65536; -const MAX_MATCH_DISTANCE = 0xffff; -const MIN_MATCH_LENGTH = 4; -const LAST_LITERALS = 5; -const MATCH_FIND_LIMIT = 12; -const HASH_LOG = 16; -const MIN_HASH_LOG = 10; -const HASH_MULTIPLIER = 2654435761; -// Encoding runs synchronously, so every block reuses one table sized for the -// largest hash log instead of allocating a fresh table per block. -const sharedHashTable = new Uint32Array(1 << HASH_LOG); -export function decodeLz4Frame(bytes, options = {}) { - const reader = new ByteReader(bytes); - decodeAssert(reader.readU32LE() === LZ4_MAGIC, "invalid LZ4 frame magic", 0); - const descriptorStart = reader.position; - const flags = reader.readU8(); - const version = (flags >>> 6) & 0x03; - decodeAssert(version === 1, "unsupported LZ4 frame version", descriptorStart); - decodeAssert((flags & 0x02) === 0, "reserved LZ4 FLG bit is set", descriptorStart); - const independentBlocks = (flags & 0x20) !== 0; - const blockChecksum = (flags & 0x10) !== 0; - const hasContentSize = (flags & 0x08) !== 0; - const contentChecksum = (flags & 0x04) !== 0; - const hasDictionary = (flags & 0x01) !== 0; - const blockDescriptor = reader.readU8(); - const maxBlockSize = decodeBlockMaximum(blockDescriptor, reader.position - 1); - let expectedContentSize; - if (hasContentSize) { - expectedContentSize = reader.readU64LE(); - } - if (hasDictionary) { - reader.readU32LE(); - throw new LoroDecodeError("LZ4 dictionaries are unsupported", descriptorStart); - } - const headerChecksumPosition = reader.position; - const storedHeaderChecksum = reader.readU8(); - if (options.checkChecksum !== false) { - const descriptor = bytes.subarray(descriptorStart, headerChecksumPosition); - const expected = (xxhash32(descriptor, 0) >>> 8) & 0xff; - decodeAssert(storedHeaderChecksum === expected, "LZ4 header checksum mismatch", headerChecksumPosition); - } - if (options.requireCanonicalProfile === true) { - decodeAssert(flags === 0x60, "noncanonical LZ4 frame flags", descriptorStart); - } - const output = new Lz4Output(); - for (;;) { - const infoOffset = reader.position; - const blockInfo = reader.readU32LE(); - if (blockInfo === 0) { - break; - } - const raw = (blockInfo & 2147483648) !== 0; - const length = blockInfo & 2147483647; - decodeAssert(length <= maxBlockSize, "LZ4 data block exceeds BD maximum", infoOffset); - const block = reader.readBytes(length); - if (blockChecksum) { - const stored = reader.readU32LE(); - if (options.checkChecksum !== false) { - decodeAssert(stored === xxhash32(block, 0), "LZ4 block checksum mismatch", reader.position - 4); - } - } - const blockOutputStart = output.length; - if (raw) { - output.ensureCapacity(block.length); - output.writeBytes(block); - } - else { - output.ensureCapacity(maxBlockSize); - const prefixStart = independentBlocks - ? blockOutputStart - : Math.max(0, blockOutputStart - WINDOW_SIZE); - decodeLz4BlockInto(block, output, prefixStart, blockOutputStart + maxBlockSize); - } - decodeAssert(output.length - blockOutputStart <= maxBlockSize, "LZ4 data block output exceeds BD maximum", infoOffset); - } - const result = output.finish(); - if (expectedContentSize !== undefined) { - decodeAssert(expectedContentSize === BigInt(result.length), "LZ4 content size mismatch", descriptorStart); - } - if (contentChecksum) { - const stored = reader.readU32LE(); - if (options.checkChecksum !== false) { - decodeAssert(stored === xxhash32(result, 0), "LZ4 content checksum mismatch", reader.position - 4); - } - } - reader.assertEnd("trailing LZ4 frame bytes"); - return result; -} -export function encodeLz4FrameRaw(bytes) { - const { descriptor, maxBlockSize } = chooseBlockDescriptor(bytes.length); - const flags = 0x60; - const header = Uint8Array.of(flags, descriptor); - const writer = new ByteWriter(bytes.length + 16 + Math.ceil(bytes.length / maxBlockSize) * 4); - writer.writeU32LE(LZ4_MAGIC); - writer.writeBytes(header); - writer.writeU8((xxhash32(header, 0) >>> 8) & 0xff); - for (let offset = 0; offset < bytes.length; offset += maxBlockSize) { - const block = bytes.subarray(offset, Math.min(bytes.length, offset + maxBlockSize)); - const compressed = encodeLz4Block(block); - if (compressed.length < block.length) { - writer.writeU32LE(compressed.length); - writer.writeBytes(compressed); - } - else { - writer.writeU32LE((2147483648 + block.length) >>> 0); - writer.writeBytes(block); - } - } - writer.writeU32LE(0); - return writer.toUint8Array(); -} -function encodeLz4Block(input) { - if (input.length < MATCH_FIND_LIMIT + 1) { - return encodeLastLiterals(input); - } - const writer = new ByteWriter(input.length); - const hashLog = Math.min(HASH_LOG, Math.max(MIN_HASH_LOG, Math.ceil(Math.log2(input.length)))); - const hashShift = 32 - hashLog; - const hashTable = sharedHashTable; - hashTable.fill(0, 0, 1 << hashLog); - const lastMatchStart = input.length - MATCH_FIND_LIMIT; - const matchLimit = input.length - LAST_LITERALS; - let anchor = 0; - let position = 0; - while (position <= lastMatchStart) { - const hash = hashSequence(input, position, hashShift); - const candidateEntry = hashTable[hash]; - hashTable[hash] = position + 1; - const candidate = candidateEntry - 1; - if (candidateEntry === 0 || - position - candidate > MAX_MATCH_DISTANCE || - !equalFourBytes(input, candidate, position)) { - position += 1; - continue; - } - let matchEnd = position + MIN_MATCH_LENGTH; - let candidateEnd = candidate + MIN_MATCH_LENGTH; - while (matchEnd < matchLimit && input[candidateEnd] === input[matchEnd]) { - matchEnd += 1; - candidateEnd += 1; - } - writeSequence(writer, input.subarray(anchor, position), position - candidate, matchEnd - position); - for (let index = position + 1; index < matchEnd; index += 1) { - if (index + MIN_MATCH_LENGTH > input.length) { - break; - } - hashTable[hashSequence(input, index, hashShift)] = index + 1; - } - position = matchEnd; - anchor = matchEnd; - } - writeLastLiterals(writer, input.subarray(anchor)); - return writer.toUint8Array(); -} -function encodeLastLiterals(input) { - const writer = new ByteWriter(input.length + 2); - writeLastLiterals(writer, input); - return writer.toUint8Array(); -} -function writeSequence(writer, literals, offset, matchLength) { - const encodedMatchLength = matchLength - MIN_MATCH_LENGTH; - writer.writeU8((Math.min(literals.length, 15) << 4) | Math.min(encodedMatchLength, 15)); - writeExtendedLength(writer, literals.length); - writer.writeBytes(literals); - writer.writeU16LE(offset); - writeExtendedLength(writer, encodedMatchLength); -} -function writeLastLiterals(writer, literals) { - writer.writeU8(Math.min(literals.length, 15) << 4); - writeExtendedLength(writer, literals.length); - writer.writeBytes(literals); -} -function writeExtendedLength(writer, length) { - if (length < 15) { - return; - } - let remaining = length - 15; - while (remaining >= 255) { - writer.writeU8(255); - remaining -= 255; - } - writer.writeU8(remaining); -} -function hashSequence(input, offset, hashShift) { - const sequence = (input[offset] | - (input[offset + 1] << 8) | - (input[offset + 2] << 16) | - (input[offset + 3] << 24)) >>> - 0; - return Math.imul(sequence, HASH_MULTIPLIER) >>> hashShift; -} -function equalFourBytes(input, left, right) { - return (input[left] === input[right] && - input[left + 1] === input[right + 1] && - input[left + 2] === input[right + 2] && - input[left + 3] === input[right + 3]); -} -function decodeLz4BlockInto(input, output, prefixStart, outputLimit) { - const reader = new ByteReader(input); - while (reader.remaining > 0) { - const token = reader.readU8(); - const literalLength = readExtendedLength(reader, token >>> 4); - decodeAssert(literalLength <= outputLimit - output.length, "LZ4 data block output exceeds BD maximum", reader.position); - output.writeBytes(reader.readBytes(literalLength)); - if (reader.remaining === 0) { - return; - } - const offsetPosition = reader.position; - const offset = reader.readU16LE(); - decodeAssert(offset !== 0, "zero LZ4 match offset", offsetPosition); - decodeAssert(offset <= output.length - prefixStart, "LZ4 match offset is out of bounds", offsetPosition); - const matchLength = 4 + readExtendedLength(reader, token & 0x0f); - decodeAssert(matchLength <= outputLimit - output.length, "LZ4 data block output exceeds BD maximum", reader.position); - // Capacity for the whole block was reserved before decoding started and - // the offset checks above keep the source range inside the output, so the - // copy can read and write the buffer directly. - const bytes = output.bytes; - let length = output.length; - if (offset >= matchLength) { - bytes.copyWithin(length, length - offset, length - offset + matchLength); - length += matchLength; - } - else { - // Overlapping matches reference bytes written by this same copy. - for (let index = 0; index < matchLength; index += 1) { - bytes[length] = bytes[length - offset]; - length += 1; - } - } - output.length = length; - } -} -class Lz4Output { - bytes = new Uint8Array(); - length = 0; - ensureCapacity(extra) { - const required = this.length + extra; - if (required <= this.bytes.length) { - return; - } - let capacity = Math.max(64, this.bytes.length); - while (capacity < required) { - capacity = Math.max(required, capacity * 2); - } - const next = new Uint8Array(capacity); - next.set(this.bytes.subarray(0, this.length)); - this.bytes = next; - } - writeBytes(bytes) { - this.ensureCapacity(bytes.length); - this.bytes.set(bytes, this.length); - this.length += bytes.length; - } - finish() { - return this.length === this.bytes.length - ? this.bytes - : this.bytes.slice(0, this.length); - } -} -function readExtendedLength(reader, base) { - let length = base; - if (base !== 15) { - return length; - } - for (;;) { - const extra = reader.readU8(); - length += extra; - decodeAssert(Number.isSafeInteger(length), "LZ4 sequence length overflow", reader.position - 1); - if (extra !== 255) { - return length; - } - } -} -function decodeBlockMaximum(descriptor, offset) { - decodeAssert((descriptor & 0x8f) === 0, "invalid LZ4 BD byte", offset); - switch ((descriptor >>> 4) & 0x07) { - case 4: - return 64 * 1024; - case 5: - return 256 * 1024; - case 6: - return 1024 * 1024; - case 7: - return 4 * 1024 * 1024; - default: - throw new LoroDecodeError("unsupported LZ4 block maximum", offset); - } -} -function chooseBlockDescriptor(length) { - if (!Number.isSafeInteger(length) || length < 0) { - throw new LoroEncodeError("invalid LZ4 input length"); - } - if (length <= 64 * 1024) { - return { descriptor: 0x40, maxBlockSize: 64 * 1024 }; - } - if (length <= 256 * 1024) { - return { descriptor: 0x50, maxBlockSize: 256 * 1024 }; - } - if (length <= 1024 * 1024) { - return { descriptor: 0x60, maxBlockSize: 1024 * 1024 }; - } - return { descriptor: 0x70, maxBlockSize: 4 * 1024 * 1024 }; -} diff --git a/loro-js/src/codec/position-arena.js b/loro-js/src/codec/position-arena.js deleted file mode 100644 index d1a5660a0..000000000 --- a/loro-js/src/codec/position-arena.js +++ /dev/null @@ -1,67 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroEncodeError, decodeAssert } from "./errors"; -import { readUlebNumber, writeUleb128 } from "./leb128"; -import { decodeAnyRleUsize, decodeColumnarVecMaybeWrapped, encodeAnyRleUsize, encodeColumnarVecWrapped, } from "./serde-columnar"; -export function decodePositionArena(bytes) { - if (bytes.length === 0) { - return []; - } - const columns = decodeColumnarVecMaybeWrapped(bytes); - decodeAssert(columns.length === 2, "position arena must have two columns"); - const commonPrefixes = decodeAnyRleUsize(columns[0]); - const reader = new ByteReader(columns[1]); - const count = readUlebNumber(reader, 10000000); - decodeAssert(count === commonPrefixes.length, "position arena column length mismatch"); - const positions = []; - let previous = new Uint8Array(); - let previousLength = 0; - for (const commonBigInt of commonPrefixes) { - // Number comparison is exact here: accepted values are at most - // previousLength (an array length), so Number(commonBigInt) is precise - // whenever the assertion passes, and larger BigInts convert to values - // greater than previousLength and fail the same way. - const common = Number(commonBigInt); - decodeAssert(common <= previousLength, "invalid position arena common prefix"); - const suffix = reader.readBytes(readUlebNumber(reader, 2147483647)); - const position = new Uint8Array(common + suffix.length); - position.set(previous.subarray(0, common)); - position.set(suffix, common); - positions.push(position); - previous = position; - previousLength = position.length; - } - reader.assertEnd("trailing position arena bytes"); - return positions; -} -export function encodePositionArena(positions, options = {}) { - if (positions.length === 0 && options.encodeEmpty !== true) { - return new Uint8Array(); - } - if (positions.length > 10000000) { - throw new LoroEncodeError(`position arena is too large: ${positions.length}`); - } - const commonPrefixes = []; - const suffixes = new ByteWriter(); - writeUleb128(suffixes, positions.length); - let previous = new Uint8Array(); - for (const position of positions) { - const common = commonPrefixLength(previous, position); - commonPrefixes.push(BigInt(common)); - const suffix = position.subarray(common); - writeUleb128(suffixes, suffix.length); - suffixes.writeBytes(suffix); - previous = position; - } - return encodeColumnarVecWrapped([ - encodeAnyRleUsize(commonPrefixes), - suffixes.toUint8Array(), - ]); -} -function commonPrefixLength(left, right) { - const length = Math.min(left.length, right.length); - let index = 0; - while (index < length && left[index] === right[index]) { - index += 1; - } - return index; -} diff --git a/loro-js/src/codec/postcard.js b/loro-js/src/codec/postcard.js deleted file mode 100644 index d2a61affa..000000000 --- a/loro-js/src/codec/postcard.js +++ /dev/null @@ -1,172 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroDecodeError, LoroEncodeError } from "./errors"; -import { I64_MAX, I64_MIN, U64_MAX, readUleb128, readUlebNumber, writeUleb128, } from "./leb128"; -const textDecoder = new TextDecoder("utf-8", { fatal: true }); -const textEncoder = new TextEncoder(); -// Shared scratch for f64 I/O, avoiding a Uint8Array + DataView allocation -// per call. Never returned to callers. -const f64Scratch = new Uint8Array(8); -const f64View = new DataView(f64Scratch.buffer); -export class PostcardReader { - input; - constructor(input) { - this.input = input instanceof ByteReader ? input : new ByteReader(input); - } - readU8() { - return this.input.readU8(); - } - readU16() { - return readUlebNumber(this.input, 0xffff); - } - readU32() { - return readUlebNumber(this.input, 4294967295); - } - readU64() { - return readUleb128(this.input, U64_MAX); - } - readUsize() { - return readUlebNumber(this.input); - } - readI32() { - const value = this.readSigned(32); - return Number(value); - } - readI64() { - return this.readSigned(64); - } - readBool() { - const value = this.input.readU8(); - if (value === 0) { - return false; - } - if (value === 1) { - return true; - } - throw new LoroDecodeError("invalid postcard boolean", this.input.position - 1); - } - readF64() { - f64Scratch.set(this.input.readBytes(8)); - return f64View.getFloat64(0, true); - } - readBytes() { - return this.input.readBytes(this.readUsize()); - } - readString() { - const offset = this.input.position; - try { - return textDecoder.decode(this.readBytes()); - } - catch { - throw new LoroDecodeError("invalid postcard UTF-8 string", offset); - } - } - readOption(readValue) { - const tag = this.input.readU8(); - if (tag === 0) { - return undefined; - } - if (tag === 1) { - return readValue(this); - } - throw new LoroDecodeError("invalid postcard option tag", this.input.position - 1); - } - readArray(readValue) { - const length = this.readUsize(); - // push() keeps the array PACKED; new Array(length) + index assignment - // would produce a permanently holey elements kind. - const values = []; - for (let index = 0; index < length; index += 1) { - values.push(readValue(this, index)); - } - return values; - } - assertEnd() { - this.input.assertEnd("trailing postcard bytes"); - } - readSigned(bits) { - const encoded = readUleb128(this.input, bits === 32 ? 0xffffffffn : U64_MAX); - const value = (encoded >> 1n) ^ -(encoded & 1n); - const min = bits === 32 ? -0x80000000n : I64_MIN; - const max = bits === 32 ? 0x7fffffffn : I64_MAX; - if (value < min || value > max) { - throw new LoroDecodeError("postcard signed integer is out of range"); - } - return value; - } -} -export class PostcardWriter { - output; - constructor(output = new ByteWriter()) { - this.output = output; - } - writeU8(value) { - this.output.writeU8(value); - } - writeU16(value) { - this.writeUnsignedNumber(value, 0xffff, "u16"); - } - writeU32(value) { - this.writeUnsignedNumber(value, 4294967295, "u32"); - } - writeU64(value) { - writeUleb128(this.output, value); - } - writeUsize(value) { - this.writeUnsignedNumber(value, Number.MAX_SAFE_INTEGER, "usize"); - } - writeI32(value) { - if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { - throw new LoroEncodeError(`i32 is out of range: ${value}`); - } - this.writeSigned(BigInt(value), 32); - } - writeI64(value) { - this.writeSigned(value, 64); - } - writeBool(value) { - this.output.writeU8(value ? 1 : 0); - } - writeF64(value) { - f64View.setFloat64(0, value, true); - this.output.writeBytes(f64Scratch); - } - writeBytes(value) { - this.writeUsize(value.length); - this.output.writeBytes(value); - } - writeString(value) { - this.writeBytes(textEncoder.encode(value)); - } - writeOption(value, writeValue) { - if (value === undefined) { - this.output.writeU8(0); - return; - } - this.output.writeU8(1); - writeValue(this, value); - } - writeArray(values, writeValue) { - this.writeUsize(values.length); - for (const value of values) { - writeValue(this, value); - } - } - toUint8Array() { - return this.output.toUint8Array(); - } - writeUnsignedNumber(value, max, label) { - if (!Number.isSafeInteger(value) || value < 0 || value > max) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } - writeUleb128(this.output, value); - } - writeSigned(value, bits) { - const min = bits === 32 ? -0x80000000n : I64_MIN; - const max = bits === 32 ? 0x7fffffffn : I64_MAX; - if (value < min || value > max) { - throw new LoroEncodeError(`i${bits} is out of range: ${value}`); - } - const encoded = (value << 1n) ^ (value >> BigInt(bits - 1)); - writeUleb128(this.output, encoded); - } -} diff --git a/loro-js/src/codec/serde-columnar.js b/loro-js/src/codec/serde-columnar.js deleted file mode 100644 index a2fa0d375..000000000 --- a/loro-js/src/codec/serde-columnar.js +++ /dev/null @@ -1,902 +0,0 @@ -import { ByteReader, ByteWriter } from "./bytes"; -import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; -import { I64_MAX, I64_MIN, U64_MAX, readUleb128, readUlebNumber, writeUleb128 } from "./leb128"; -const U32_MAX = 4294967295; -const I32_MIN = -2147483648; -const I32_MAX = 2147483647; -const I128_MIN = -(1n << 127n); -const I128_MAX = (1n << 127n) - 1n; -const U128_MAX = (1n << 128n) - 1n; -const MAX_COLUMN_VALUES = 10000000; -const MIN_SAFE_BIGINT = BigInt(-Number.MAX_SAFE_INTEGER); -const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); -export function decodeColumnarVec(bytes) { - const reader = new ByteReader(bytes); - const columns = readColumnarVec(reader); - reader.assertEnd("trailing serde-columnar bytes"); - return columns; -} -export function takeColumnarVec(bytes) { - const reader = new ByteReader(bytes); - const columns = readColumnarVec(reader); - return [columns, bytes.subarray(reader.position)]; -} -export function decodeColumnarVecMaybeWrapped(bytes) { - if (bytes.length === 0) { - return []; - } - const reader = new ByteReader(bytes); - if (readUlebNumber(reader, MAX_COLUMN_VALUES) === 1) { - const columns = readColumnarVec(reader); - reader.assertEnd("trailing wrapped serde-columnar bytes"); - return columns; - } - return decodeColumnarVec(bytes); -} -export function encodeColumnarVec(columns) { - const writer = new ByteWriter(); - writeUleb128(writer, columns.length); - for (const column of columns) { - writeUleb128(writer, column.length); - writer.writeBytes(column); - } - return writer.toUint8Array(); -} -export function encodeColumnarVecWrapped(columns) { - const writer = new ByteWriter(); - writeUleb128(writer, 1); - writer.writeBytes(encodeColumnarVec(columns)); - return writer.toUint8Array(); -} -export function decodeBoolRle(bytes) { - const reader = new ByteReader(bytes); - const values = []; - let state = false; - while (reader.remaining > 0) { - const length = readUlebNumber(reader, MAX_COLUMN_VALUES); - assertDecodedLength(values.length, length, "bool RLE"); - for (let index = 0; index < length; index += 1) { - values.push(state); - } - state = !state; - } - return values; -} -export function encodeBoolRle(values) { - const writer = new ByteWriter(); - if (values.length === 0) { - return writer.toUint8Array(); - } - let state = false; - let runLength = 0; - for (const value of values) { - if (value === state) { - runLength += 1; - } - else { - writeUleb128(writer, runLength); - state = !state; - runLength = 1; - } - } - writeUleb128(writer, runLength); - return writer.toUint8Array(); -} -export function takeBoolRle(bytes, count) { - assertCount(count); - const reader = new ByteReader(bytes); - const values = []; - let state = false; - while (values.length < count) { - decodeAssert(reader.remaining > 0, "bool RLE has too few elements", reader.position); - const length = readUlebNumber(reader, MAX_COLUMN_VALUES); - decodeAssert(values.length + length <= count, "bool RLE has too many elements"); - for (let index = 0; index < length; index += 1) { - values.push(state); - } - state = !state; - } - return [values, bytes.subarray(reader.position)]; -} -export function decodeRleU8(bytes) { - return decodeAnyRle(bytes, readPostcardU8); -} -export function encodeRleU8(values) { - return encodeAnyRleLiteral(values, writePostcardU8); -} -export function decodeRleU32(bytes) { - return decodeAnyRle(bytes, readPostcardU32); -} -export function encodeRleU32(values) { - return encodeAnyRleLiteral(values, writePostcardU32); -} -export function decodeAnyRleU32(bytes) { - return decodeAnyRle(bytes, readPostcardU32); -} -export function encodeAnyRleU32(values) { - return encodeAnyRleLiteral(values, writePostcardU32); -} -export function takeAnyRleU32(bytes, count) { - return takeAnyRle(bytes, count, readPostcardU32); -} -export function decodeAnyRleU64(bytes) { - return decodeAnyRle(bytes, readPostcardU64); -} -export function encodeAnyRleU64(values) { - return encodeAnyRleLiteral(values, writePostcardU64); -} -export function decodeAnyRleUsize(bytes) { - return decodeAnyRleU64(bytes); -} -export function encodeAnyRleUsize(values) { - return encodeAnyRleU64(values); -} -export function takeAnyRleUsize(bytes, count) { - return takeAnyRle(bytes, count, readPostcardU64); -} -export function decodeAnyRleI32(bytes) { - return decodeAnyRle(bytes, readPostcardI32); -} -export function encodeAnyRleI32(values) { - return encodeAnyRleLiteral(values, writePostcardI32); -} -export function takeAnyRleI32(bytes, count) { - return takeAnyRle(bytes, count, readPostcardI32); -} -export function decodeDeltaRleU32(bytes) { - return decodeDeltaNumber(bytes, 0, U32_MAX); -} -export function encodeDeltaRleU32(values) { - return encodeDeltaNumber(values, assertU32); -} -export function decodeDeltaRleI32(bytes) { - return decodeDeltaNumber(bytes, I32_MIN, I32_MAX); -} -export function encodeDeltaRleI32(values) { - return encodeDeltaNumber(values, assertI32); -} -export function decodeDeltaRleUsize(bytes) { - return decodeDelta(bytes, 0n, U64_MAX, identity); -} -export function encodeDeltaRleUsize(values) { - return encodeDelta(values, assertUsize); -} -export function decodeDeltaRleIsize(bytes) { - return decodeDelta(bytes, I64_MIN, I64_MAX, identity); -} -export function encodeDeltaRleIsize(values) { - return encodeDelta(values, assertIsize); -} -export function decodeDeltaOfDeltaI64(bytes) { - const reader = new ByteReader(bytes); - const first = readPostcardOptionalI64(reader); - decodeAssert(reader.remaining >= 1, "invalid delta-of-delta bytes", reader.position); - const lastUsedBits = reader.readU8(); - const bitstream = reader.readRemaining(); - if (first === undefined) { - decodeAssert(lastUsedBits === 0 && bitstream.length === 0, "invalid empty delta-of-delta encoding"); - return []; - } - const bits = BitReader.forCompleteStream(bitstream, lastUsedBits); - const values = [first]; - let previous = first; - let delta = 0n; - // Number fast path: while every intermediate value stays a safe integer, - // accumulate with plain number arithmetic and box only the emitted element. - let previousNumber = 0; - let deltaNumber = 0; - let useNumber = first >= MIN_SAFE_BIGINT && first <= MAX_SAFE_BIGINT; - if (useNumber) { - previousNumber = Number(first); - } - while (bits.remaining > 0) { - const step = decodeDeltaOfDeltaValue(bits); - if (useNumber) { - if (typeof step === "number") { - const nextDelta = deltaNumber + step; - const next = previousNumber + nextDelta; - if (Number.isSafeInteger(nextDelta) && Number.isSafeInteger(next)) { - deltaNumber = nextDelta; - previousNumber = next; - values.push(BigInt(next)); - continue; - } - } - delta = BigInt(deltaNumber); - previous = BigInt(previousNumber); - useNumber = false; - } - const stepBigInt = typeof step === "number" ? BigInt(step) : step; - delta = checkedI64(delta + stepBigInt, "delta-of-delta delta"); - previous = checkedI64(previous + delta, "delta-of-delta value"); - values.push(previous); - } - return values; -} -export function takeDeltaOfDeltaI64(bytes, count) { - assertCount(count); - const reader = new ByteReader(bytes); - const first = readPostcardOptionalI64(reader); - decodeAssert(reader.remaining >= 1, "invalid delta-of-delta bytes", reader.position); - const lastUsedBits = reader.readU8(); - const bitstreamOffset = reader.position; - if (first === undefined) { - decodeAssert(count === 0, "delta-of-delta has too few elements"); - decodeAssert(lastUsedBits === 0, "invalid empty delta-of-delta encoding"); - return [[], bytes.subarray(bitstreamOffset)]; - } - decodeAssert(count > 0, "delta-of-delta has too many elements"); - const bits = BitReader.forPrefix(bytes.subarray(bitstreamOffset)); - const values = [first]; - let previous = first; - let delta = 0n; - // Number fast path, mirroring decodeDeltaOfDeltaI64. - let previousNumber = 0; - let deltaNumber = 0; - let useNumber = first >= MIN_SAFE_BIGINT && first <= MAX_SAFE_BIGINT; - if (useNumber) { - previousNumber = Number(first); - } - while (values.length < count) { - const step = decodeDeltaOfDeltaValue(bits); - if (useNumber) { - if (typeof step === "number") { - const nextDelta = deltaNumber + step; - const next = previousNumber + nextDelta; - if (Number.isSafeInteger(nextDelta) && Number.isSafeInteger(next)) { - deltaNumber = nextDelta; - previousNumber = next; - values.push(BigInt(next)); - continue; - } - } - delta = BigInt(deltaNumber); - previous = BigInt(previousNumber); - useNumber = false; - } - const stepBigInt = typeof step === "number" ? BigInt(step) : step; - delta = checkedI64(delta + stepBigInt, "delta-of-delta delta"); - previous = checkedI64(previous + delta, "delta-of-delta value"); - values.push(previous); - } - if (count === 1) { - decodeAssert(lastUsedBits === 0, "invalid single-value delta-of-delta encoding"); - } - else { - const expectedLastUsed = bits.position % 8 || 8; - decodeAssert(lastUsedBits === expectedLastUsed, "delta-of-delta last-used-bits mismatch"); - } - return [values, bytes.subarray(bitstreamOffset + Math.ceil(bits.position / 8))]; -} -export function encodeDeltaOfDeltaI64(values) { - const writer = new ByteWriter(); - if (values.length === 0) { - writePostcardOptionalI64(writer, undefined); - writer.writeU8(0); - return writer.toUint8Array(); - } - for (const value of values) { - assertBigIntRange(value, I64_MIN, I64_MAX, "i64"); - } - writePostcardOptionalI64(writer, values[0]); - if (values.length === 1) { - writer.writeU8(0); - return writer.toUint8Array(); - } - const bits = new BitWriter(); - let previousDelta = 0n; - // Number fast path: differences of safe integers stay exact while the - // running delta and delta-of-delta remain safe integers. - let previousDeltaNumber = 0; - let useNumber = true; - for (let index = 1; index < values.length; index += 1) { - const current = values[index]; - const prior = values[index - 1]; - if (useNumber) { - if (current >= MIN_SAFE_BIGINT && - current <= MAX_SAFE_BIGINT && - prior >= MIN_SAFE_BIGINT && - prior <= MAX_SAFE_BIGINT) { - const deltaNumber = Number(current) - Number(prior); - const difference = deltaNumber - previousDeltaNumber; - if (Number.isSafeInteger(deltaNumber) && Number.isSafeInteger(difference)) { - previousDeltaNumber = deltaNumber; - encodeDeltaOfDeltaValueNumber(bits, difference); - continue; - } - } - previousDelta = BigInt(previousDeltaNumber); - useNumber = false; - } - const delta = checkedI64(current - prior, "delta-of-delta delta"); - const deltaOfDelta = checkedI64(delta - previousDelta, "delta-of-delta difference"); - previousDelta = delta; - encodeDeltaOfDeltaValue(bits, deltaOfDelta); - } - const encoded = bits.finish(); - writer.writeU8(encoded.lastUsedBits); - writer.writeBytes(encoded.bytes); - return writer.toUint8Array(); -} -function readColumnarVec(reader) { - const count = readUlebNumber(reader, MAX_COLUMN_VALUES); - const columns = []; - for (let index = 0; index < count; index += 1) { - columns.push(reader.readBytes(readUlebNumber(reader, 2147483647))); - } - return columns; -} -function decodeAnyRle(bytes, readValue) { - const reader = new ByteReader(bytes); - const values = []; - while (reader.remaining > 0) { - readAnyRleSegment(reader, values, MAX_COLUMN_VALUES, readValue); - } - return values; -} -function takeAnyRle(bytes, count, readValue) { - assertCount(count); - const reader = new ByteReader(bytes); - const values = []; - while (values.length < count) { - decodeAssert(reader.remaining > 0, "AnyRle has too few elements", reader.position); - readAnyRleSegment(reader, values, count, readValue); - } - return [values, bytes.subarray(reader.position)]; -} -function readAnyRleSegment(reader, values, limit, readValue) { - const signedLength = readPostcardI64(reader); - decodeAssert(signedLength !== 0n, "AnyRle segment has zero length", reader.position); - const lengthBigInt = signedLength < 0n ? -signedLength : signedLength; - decodeAssert(lengthBigInt <= BigInt(MAX_COLUMN_VALUES), "AnyRle segment is too long"); - const length = Number(lengthBigInt); - decodeAssert(values.length + length <= limit, "AnyRle has too many elements"); - if (signedLength > 0n) { - const value = readValue(reader); - for (let index = 0; index < length; index += 1) { - values.push(value); - } - return; - } - for (let index = 0; index < length; index += 1) { - values.push(readValue(reader)); - } -} -function encodeAnyRleLiteral(values, writeValue) { - const writer = new ByteWriter(); - if (values.length === 0) { - return writer.toUint8Array(); - } - if (values.length > MAX_COLUMN_VALUES) { - throw new LoroEncodeError(`AnyRle input is too long: ${values.length}`); - } - writePostcardI64(writer, -BigInt(values.length)); - for (const value of values) { - writeValue(writer, value); - } - return writer.toUint8Array(); -} -function encodeDelta(values, toBigInt) { - const writer = new ByteWriter(); - if (values.length === 0) { - return writer.toUint8Array(); - } - if (values.length > MAX_COLUMN_VALUES) { - throw new LoroEncodeError(`AnyRle input is too long: ${values.length}`); - } - writePostcardI64(writer, -BigInt(values.length)); - let previous = 0n; - // Number fast path: safe-integer values encode through the number i128 - // writer, avoiding per-element BigInt subtraction and comparison. - let previousNumber = 0; - let useNumber = true; - for (const input of values) { - const value = toBigInt(input); - if (useNumber) { - if (value >= MIN_SAFE_BIGINT && value <= MAX_SAFE_BIGINT) { - const valueNumber = Number(value); - const deltaNumber = valueNumber - previousNumber; - if (Number.isSafeInteger(deltaNumber)) { - writePostcardI128Number(writer, deltaNumber); - previousNumber = valueNumber; - continue; - } - } - previous = BigInt(previousNumber); - useNumber = false; - } - const delta = value - previous; - assertBigIntRange(delta, I128_MIN, I128_MAX, "delta RLE delta"); - writePostcardI128(writer, delta); - previous = value; - } - return writer.toUint8Array(); -} -function decodeDelta(bytes, min, max, fromBigInt) { - const reader = new ByteReader(bytes); - const values = []; - let value = 0n; - // Number fast path: accumulate with plain number arithmetic while the - // running value stays a safe integer. A safe integer always compares - // exactly against these bounds: any bound outside the safe range is - // replaced by the nearest safe-integer limit, which accepts/rejects the - // same values as the BigInt comparison. - let valueNumber = 0; - let useNumber = true; - const minNumber = min < MIN_SAFE_BIGINT ? Number.MIN_SAFE_INTEGER : Number(min); - const maxNumber = max > MAX_SAFE_BIGINT ? Number.MAX_SAFE_INTEGER : Number(max); - const append = (delta) => { - if (useNumber) { - if (typeof delta === "number") { - const next = valueNumber + delta; - if (Number.isSafeInteger(next)) { - valueNumber = next; - decodeAssert(next >= minNumber && next <= maxNumber, "delta RLE value is out of range"); - values.push(fromBigInt(BigInt(next))); - return; - } - } - value = BigInt(valueNumber); - useNumber = false; - } - const deltaBigInt = typeof delta === "number" ? BigInt(delta) : delta; - value += deltaBigInt; - decodeAssert(value >= min && value <= max, "delta RLE value is out of range"); - values.push(fromBigInt(value)); - }; - while (reader.remaining > 0) { - const signedLength = readPostcardI64(reader); - decodeAssert(signedLength !== 0n, "AnyRle segment has zero length", reader.position); - const lengthBigInt = signedLength < 0n ? -signedLength : signedLength; - decodeAssert(lengthBigInt <= BigInt(MAX_COLUMN_VALUES), "AnyRle segment is too long"); - const length = Number(lengthBigInt); - decodeAssert(values.length + length <= MAX_COLUMN_VALUES, "AnyRle has too many elements"); - if (signedLength > 0n) { - const delta = readPostcardI128NumberOrBigInt(reader); - for (let index = 0; index < length; index += 1) { - append(delta); - } - } - else { - for (let index = 0; index < length; index += 1) { - append(readPostcardI128NumberOrBigInt(reader)); - } - } - } - return values; -} -function encodeDeltaNumber(values, validate) { - const writer = new ByteWriter(); - if (values.length === 0) { - return writer.toUint8Array(); - } - if (values.length > MAX_COLUMN_VALUES) { - throw new LoroEncodeError(`AnyRle input is too long: ${values.length}`); - } - writePostcardI64(writer, -BigInt(values.length)); - let previous = 0; - for (const input of values) { - const value = validate(input); - writePostcardI128Number(writer, value - previous); - previous = value; - } - return writer.toUint8Array(); -} -function decodeDeltaNumber(bytes, min, max) { - const reader = new ByteReader(bytes); - const values = []; - let value = 0; - const append = (delta) => { - value += delta; - decodeAssert(value >= min && value <= max, "delta RLE value is out of range"); - values.push(value); - }; - while (reader.remaining > 0) { - const signedLength = readPostcardI64(reader); - decodeAssert(signedLength !== 0n, "AnyRle segment has zero length", reader.position); - const lengthBigInt = signedLength < 0n ? -signedLength : signedLength; - decodeAssert(lengthBigInt <= BigInt(MAX_COLUMN_VALUES), "AnyRle segment is too long"); - const length = Number(lengthBigInt); - decodeAssert(values.length + length <= MAX_COLUMN_VALUES, "AnyRle has too many elements"); - if (signedLength > 0n) { - const delta = readPostcardI128Number(reader); - for (let index = 0; index < length; index += 1) { - append(delta); - } - } - else { - for (let index = 0; index < length; index += 1) { - append(readPostcardI128Number(reader)); - } - } - } - return values; -} -function readPostcardU8(reader) { - return reader.readU8(); -} -function writePostcardU8(writer, value) { - writer.writeU8(value); -} -function readPostcardU32(reader) { - return readUlebNumber(reader, 4294967295); -} -function writePostcardU32(writer, value) { - if (!Number.isSafeInteger(value) || value < 0 || value > 4294967295) { - throw new LoroEncodeError(`u32 is out of range: ${value}`); - } - writeUleb128(writer, value); -} -function readPostcardU64(reader) { - return readUleb128(reader, U64_MAX); -} -function writePostcardU64(writer, value) { - writeUleb128(writer, value); -} -function readPostcardI32(reader) { - const encoded = readUlebNumber(reader, 4294967295); - // ZigZag decode; every u32-encoded value lands in the i32 range. - const value = (encoded >>> 1) ^ -(encoded & 1); - if (value < I32_MIN || value > I32_MAX) { - throw new LoroDecodeError("postcard signed integer is out of range"); - } - return value; -} -function writePostcardI32(writer, value) { - if (!Number.isSafeInteger(value) || value < I32_MIN || value > I32_MAX) { - throw new LoroEncodeError(`i32 is out of range: ${value}`); - } - writeUleb128(writer, ((value << 1) ^ (value >> 31)) >>> 0); -} -function readPostcardI64(reader) { - const encoded = readUleb128(reader, U64_MAX); - const value = (encoded >> 1n) ^ -(encoded & 1n); - if (value < I64_MIN || value > I64_MAX) { - throw new LoroDecodeError("postcard signed integer is out of range"); - } - return value; -} -function writePostcardI64(writer, value) { - if (value < I64_MIN || value > I64_MAX) { - throw new LoroEncodeError(`i64 is out of range: ${value}`); - } - writeUleb128(writer, (value << 1n) ^ (value >> 63n)); -} -function readPostcardOptionalI64(reader) { - const tag = readPostcardU64(reader); - if (tag === 0n) { - return undefined; - } - if (tag === 1n) { - return readPostcardI64(reader); - } - throw new LoroDecodeError("invalid postcard Option tag", reader.position); -} -function writePostcardOptionalI64(writer, value) { - writePostcardU64(writer, value === undefined ? 0n : 1n); - if (value !== undefined) { - writePostcardI64(writer, value); - } -} -function writePostcardI128(writer, value) { - assertBigIntRange(value, I128_MIN, I128_MAX, "i128"); - const encoded = value >= 0n ? value << 1n : (-value << 1n) - 1n; - writeVarintU128(writer, encoded); -} -function readPostcardI128Number(reader) { - const start = reader.position; - let encoded = 0; - let multiplier = 1; - for (let index = 0; index < 19; index += 1) { - const byte = reader.readU8(); - const payload = byte & 0x7f; - if (payload !== 0 && multiplier > Number.MAX_SAFE_INTEGER / payload) { - throw new LoroDecodeError("i128 varint exceeds safe integer range", start); - } - encoded += payload * multiplier; - if (!Number.isSafeInteger(encoded)) { - throw new LoroDecodeError("i128 varint exceeds safe integer range", start); - } - if ((byte & 0x80) === 0) { - if (encoded < 2147483648) { - // Smi ZigZag decode; identical to the division-based formula below. - return (encoded >>> 1) ^ -(encoded & 1); - } - return encoded % 2 === 0 ? encoded / 2 : -(encoded + 1) / 2; - } - multiplier *= 128; - } - throw new LoroDecodeError("u128 varint overflow", start); -} -function writePostcardI128Number(writer, value) { - if (Number.isSafeInteger(value) && value > -2147483648 && value < 2147483648) { - // Smi-only ZigZag encode and varint emission for 31-bit values. - let encoded = ((value << 1) ^ (value >> 31)) >>> 0; - do { - let byte = encoded & 0x7f; - encoded >>>= 7; - if (encoded !== 0) { - byte |= 0x80; - } - writer.writeU8(byte); - } while (encoded !== 0); - return; - } - const encoded = value >= 0 ? value * 2 : -value * 2 - 1; - let remaining = encoded; - do { - let byte = remaining % 128; - remaining = Math.floor(remaining / 128); - if (remaining !== 0) { - byte |= 0x80; - } - writer.writeU8(byte); - } while (remaining !== 0); -} -// Reads a postcard i128, keeping the whole decode in number arithmetic while -// the encoded varint stays a safe integer and falling back to BigInt only for -// larger (still valid u128) inputs. -function readPostcardI128NumberOrBigInt(reader) { - const start = reader.position; - let encoded = 0; - let multiplier = 1; - let encodedBigInt = 0n; - let useBigInt = false; - for (let index = 0; index < 19; index += 1) { - const byte = reader.readU8(); - const payload = byte & 0x7f; - if (index === 18 && payload > 3) { - throw new LoroDecodeError("u128 varint overflow", start); - } - if (useBigInt) { - encodedBigInt |= BigInt(payload) << BigInt(index * 7); - } - else if (payload !== 0) { - if (multiplier <= Number.MAX_SAFE_INTEGER / payload) { - const sum = encoded + payload * multiplier; - if (Number.isSafeInteger(sum)) { - encoded = sum; - } - else { - useBigInt = true; - encodedBigInt = BigInt(encoded) | (BigInt(payload) << BigInt(index * 7)); - } - } - else { - useBigInt = true; - encodedBigInt = BigInt(encoded) | (BigInt(payload) << BigInt(index * 7)); - } - } - if ((byte & 0x80) === 0) { - if (!useBigInt) { - if (encoded < 2147483648) { - return (encoded >>> 1) ^ -(encoded & 1); - } - return encoded % 2 === 0 ? encoded / 2 : -(encoded + 1) / 2; - } - const value = (encodedBigInt >> 1n) ^ -(encodedBigInt & 1n); - decodeAssert(value >= I128_MIN && value <= I128_MAX, "postcard i128 is out of range"); - return value; - } - multiplier *= 128; - } - throw new LoroDecodeError("u128 varint overflow", start); -} -function writeVarintU128(writer, input) { - assertBigIntRange(input, 0n, U128_MAX, "u128"); - let value = input; - do { - let byte = Number(value & 0x7fn); - value >>= 7n; - if (value !== 0n) { - byte |= 0x80; - } - writer.writeU8(byte); - } while (value !== 0n); -} -class BitReader { - #bytes; - #totalBits; - #bytePosition = 0; - // Pending bits read from the stream but not yet consumed, right-aligned. - // The invariant #bitCount <= 7 holds between read calls, so every - // intermediate below stays a positive 31-bit value and bitwise ops are - // exact Smi operations. - #bitBuffer = 0; - #bitCount = 0; - constructor(bytes, totalBits) { - this.#bytes = bytes; - this.#totalBits = totalBits; - } - static forCompleteStream(bytes, lastUsedBits) { - decodeAssert(lastUsedBits >= 0 && lastUsedBits <= 8, "invalid last-used-bits value"); - if (lastUsedBits === 0) { - decodeAssert(bytes.length === 0, "unexpected delta-of-delta bitstream bytes"); - return new BitReader(bytes, 0); - } - decodeAssert(bytes.length > 0, "missing delta-of-delta bitstream bytes"); - return new BitReader(bytes, (bytes.length - 1) * 8 + lastUsedBits); - } - static forPrefix(bytes) { - return new BitReader(bytes, bytes.length * 8); - } - get position() { - return this.#bytePosition * 8 - this.#bitCount; - } - get remaining() { - return this.#totalBits - this.position; - } - // Reads up to 24 bits (the refill loop would exceed 31-bit intermediate - // values above that) and returns them as a number. Bits are consumed - // MSB-first, matching the original bit-at-a-time implementation. - readBitsNumber(count) { - decodeAssert(Number.isSafeInteger(count) && count >= 0 && count <= 24, "invalid bit width"); - decodeAssert(this.remaining >= count, "unexpected end of delta-of-delta bitstream"); - while (this.#bitCount < count) { - this.#bitBuffer = (this.#bitBuffer << 8) | this.#bytes[this.#bytePosition]; - this.#bytePosition += 1; - this.#bitCount += 8; - } - const shift = this.#bitCount - count; - const value = this.#bitBuffer >>> shift; - this.#bitBuffer &= (1 << shift) - 1; - this.#bitCount -= count; - return value; - } - readBitsBigInt(count) { - decodeAssert(Number.isSafeInteger(count) && count >= 0 && count <= 64, "invalid bit width"); - decodeAssert(this.remaining >= count, "unexpected end of delta-of-delta bitstream"); - let value = 0n; - let remaining = count; - while (remaining > 0) { - const step = Math.min(remaining, 24); - value = (value << BigInt(step)) | BigInt(this.readBitsNumber(step)); - remaining -= step; - } - return value; - } -} -class BitWriter { - #bytes = []; - // Same right-aligned pending-bits representation as BitReader, with the - // invariant #bitCount <= 7 between write calls. - #bitBuffer = 0; - #bitCount = 0; - // Writes the low `count` bits of value, MSB first. Like the original - // bit-at-a-time loop, bits of value above `count` are ignored. - writeBitsNumber(value, count) { - encodeAssert(Number.isSafeInteger(count) && count >= 0 && count <= 24, "invalid bit width"); - this.#bitBuffer = (this.#bitBuffer << count) | (value & ((1 << count) - 1)); - this.#bitCount += count; - while (this.#bitCount >= 8) { - const shift = this.#bitCount - 8; - this.#bytes.push(this.#bitBuffer >>> shift); - this.#bitBuffer &= (1 << shift) - 1; - this.#bitCount -= 8; - } - } - writeBitsBigInt(value, count) { - encodeAssert(count >= 0 && count <= 64, "invalid bit width"); - let remaining = count; - while (remaining > 0) { - const step = Math.min(remaining, 24); - remaining -= step; - this.writeBitsNumber(Number((value >> BigInt(remaining)) & 0xffffffn), step); - } - } - finish() { - if (this.#bitCount === 0) { - return { bytes: Uint8Array.from(this.#bytes), lastUsedBits: 8 }; - } - this.#bytes.push(this.#bitBuffer << (8 - this.#bitCount)); - return { bytes: Uint8Array.from(this.#bytes), lastUsedBits: this.#bitCount }; - } -} -function decodeDeltaOfDeltaValue(reader) { - if (reader.readBitsNumber(1) === 0) { - return 0; - } - if (reader.readBitsNumber(1) === 0) { - return reader.readBitsNumber(7) - 63; - } - if (reader.readBitsNumber(1) === 0) { - return reader.readBitsNumber(9) - 255; - } - if (reader.readBitsNumber(1) === 0) { - return reader.readBitsNumber(12) - 2047; - } - if (reader.readBitsNumber(1) === 0) { - return reader.readBitsNumber(21) - 1048575; - } - return BigInt.asIntN(64, reader.readBitsBigInt(64)); -} -function encodeDeltaOfDeltaValue(writer, value) { - if (value === 0n) { - writer.writeBitsNumber(0, 1); - } - else if (value >= -63n && value <= 64n) { - writer.writeBitsNumber(0b10, 2); - writer.writeBitsNumber(Number(value + 63n), 7); - } - else if (value >= -255n && value <= 256n) { - writer.writeBitsNumber(0b110, 3); - writer.writeBitsNumber(Number(value + 255n), 9); - } - else if (value >= -2047n && value <= 2048n) { - writer.writeBitsNumber(0b1110, 4); - writer.writeBitsNumber(Number(value + 2047n), 12); - } - else if (value >= -1048575n && value <= 1048576n) { - writer.writeBitsNumber(0b11110, 5); - writer.writeBitsNumber(Number(value + 1048575n), 21); - } - else { - writer.writeBitsNumber(0b11111, 5); - writer.writeBitsBigInt(BigInt.asUintN(64, value), 64); - } -} -function encodeDeltaOfDeltaValueNumber(writer, value) { - if (value === 0) { - writer.writeBitsNumber(0, 1); - } - else if (value >= -63 && value <= 64) { - writer.writeBitsNumber(0b10, 2); - writer.writeBitsNumber(value + 63, 7); - } - else if (value >= -255 && value <= 256) { - writer.writeBitsNumber(0b110, 3); - writer.writeBitsNumber(value + 255, 9); - } - else if (value >= -2047 && value <= 2048) { - writer.writeBitsNumber(0b1110, 4); - writer.writeBitsNumber(value + 2047, 12); - } - else if (value >= -1048575 && value <= 1048576) { - writer.writeBitsNumber(0b11110, 5); - writer.writeBitsNumber(value + 1048575, 21); - } - else { - writer.writeBitsNumber(0b11111, 5); - writer.writeBitsBigInt(BigInt.asUintN(64, BigInt(value)), 64); - } -} -function checkedI64(value, label) { - decodeAssert(value >= I64_MIN && value <= I64_MAX, `${label} is out of range`); - return value; -} -function assertDecodedLength(current, extra, label) { - decodeAssert(current + extra <= MAX_COLUMN_VALUES, `${label} output is too long`); -} -function assertCount(count) { - if (!Number.isSafeInteger(count) || count < 0 || count > MAX_COLUMN_VALUES) { - throw new LoroDecodeError(`invalid requested column value count: ${count}`); - } -} -function assertU32(value) { - if (!Number.isSafeInteger(value) || value < 0 || value > 4294967295) { - throw new LoroEncodeError(`u32 is out of range: ${value}`); - } - return value; -} -function assertI32(value) { - if (!Number.isSafeInteger(value) || value < -2147483648 || value > 2147483647) { - throw new LoroEncodeError(`i32 is out of range: ${value}`); - } - return value; -} -function assertUsize(value) { - assertBigIntRange(value, 0n, U64_MAX, "usize"); - return value; -} -function assertIsize(value) { - assertBigIntRange(value, I64_MIN, I64_MAX, "isize"); - return value; -} -function identity(value) { - return value; -} -function assertBigIntRange(value, min, max, label) { - if (value < min || value > max) { - throw new LoroEncodeError(`${label} is out of range: ${value}`); - } -} diff --git a/loro-js/src/codec/sstable.js b/loro-js/src/codec/sstable.js deleted file mode 100644 index 3194a803f..000000000 --- a/loro-js/src/codec/sstable.js +++ /dev/null @@ -1,261 +0,0 @@ -import { ByteReader, ByteWriter, bytesEqual, compareBytes } from "./bytes"; -import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; -import { decodeLz4Frame, encodeLz4FrameRaw } from "./lz4"; -import { LORO_XXHASH_SEED, xxhash32 } from "./xxhash32"; -const SSTABLE_MAGIC = Uint8Array.of(0x4c, 0x4f, 0x52, 0x4f); -const SSTABLE_SCHEMA = 0; -const MAX_BLOCK_COUNT = 10000000; -export function decodeSstable(bytes, options = {}) { - if (bytes.length === 0) { - return []; - } - decodeAssert(bytes.length >= 17, "SSTable is too short", 0); - decodeAssert(bytesEqual(bytes.subarray(0, 4), SSTABLE_MAGIC), "invalid SSTable magic", 0); - decodeAssert(bytes[4] === SSTABLE_SCHEMA, "unsupported SSTable schema", 4); - const footer = new ByteReader(bytes, bytes.length - 4, 4); - const metadataOffset = footer.readU32LE(); - decodeAssert(metadataOffset >= 5 && metadataOffset < bytes.length - 4, "invalid SSTable metadata offset", bytes.length - 4); - const metadataBytes = bytes.subarray(metadataOffset, bytes.length - 4); - const metadata = decodeMetadata(metadataBytes, options.checkChecksum !== false); - const entries = []; - for (let index = 0; index < metadata.length; index += 1) { - const current = metadata[index]; - const end = index + 1 < metadata.length ? metadata[index + 1].offset : metadataOffset; - decodeAssert(current.offset >= 5, "invalid SSTable block offset", current.offset); - decodeAssert(end > current.offset && end <= metadataOffset, "invalid SSTable block range", current.offset); - const stored = bytes.subarray(current.offset, end); - decodeAssert(stored.length >= 4, "SSTable block lacks checksum", current.offset); - const payload = stored.subarray(0, stored.length - 4); - const checksum = new ByteReader(stored, stored.length - 4, 4).readU32LE(); - if (options.checkChecksum !== false) { - decodeAssert(checksum === xxhash32(payload, LORO_XXHASH_SEED), "SSTable block checksum mismatch", end - 4); - } - const decoded = current.compression === 0 ? payload : decodeLz4Frame(payload, options); - if (current.large) { - entries.push({ key: current.firstKey, value: decoded }); - continue; - } - const blockEntries = decodeNormalBlock(decoded, current); - entries.push(...blockEntries); - } - validateEntryOrder(entries); - return entries; -} -export function encodeSstable(input, options = {}) { - if (input.length === 0) { - return new Uint8Array(); - } - const blockSize = options.blockSize ?? 4096; - const compression = options.compression ?? "auto"; - if (!Number.isSafeInteger(blockSize) || blockSize <= 0 || blockSize > 0xffff) { - throw new LoroEncodeError(`invalid SSTable block size ${blockSize}`); - } - // Sorting only needs to reorder references; keys and values are copied into - // the output while encoding and never retained past the returned bytes. - const entries = input.slice(); - entries.sort((left, right) => compareBytes(left.key, right.key)); - validateEntriesForEncoding(entries); - const blocks = []; - for (let index = 0; index < entries.length;) { - const first = entries[index]; - if (first.value.length > blockSize || first.value.length > 0xffff) { - blocks.push(encodeStoredBlock(first.value, true, first.key, undefined, compression)); - index += 1; - continue; - } - const data = new ByteWriter(); - const offsets = [0]; - const firstKey = first.key; - let lastKey = firstKey; - data.writeBytes(first.value); - index += 1; - while (index < entries.length) { - const next = entries[index]; - if (next.value.length > blockSize || next.value.length > 0xffff) { - break; - } - const common = commonPrefixLength(firstKey, next.key); - const suffixLength = next.key.length - common; - if (suffixLength > 0xffff || data.length > 0xffff) { - break; - } - const added = 3 + suffixLength + next.value.length; - const prospective = data.length + added + (offsets.length + 1) * 2 + 2; - if (prospective > blockSize) { - break; - } - offsets.push(data.length); - data.writeU8(common); - data.writeU16LE(suffixLength); - data.writeBytes(next.key.subarray(common)); - data.writeBytes(next.value); - lastKey = next.key; - index += 1; - } - const body = new ByteWriter(data.length + offsets.length * 2 + 2); - body.writeBytes(data.toUint8Array()); - for (const offset of offsets) { - body.writeU16LE(offset); - } - body.writeU16LE(offsets.length); - blocks.push(encodeStoredBlock(body.toUint8Array(), false, firstKey, lastKey, compression)); - } - return encodeTable(blocks); -} -function decodeMetadata(bytes, checkChecksum) { - decodeAssert(bytes.length >= 8, "SSTable metadata is too short"); - const reader = new ByteReader(bytes); - const count = reader.readU32LE(); - decodeAssert(count > 0 && count <= MAX_BLOCK_COUNT, "invalid SSTable block count", 0); - if (checkChecksum) { - const stored = new ByteReader(bytes, bytes.length - 4, 4).readU32LE(); - const expected = xxhash32(bytes.subarray(4, bytes.length - 4), LORO_XXHASH_SEED); - decodeAssert(stored === expected, "SSTable metadata checksum mismatch", bytes.length - 4); - } - const metadata = []; - for (let index = 0; index < count; index += 1) { - const offset = reader.readU32LE(); - const firstKey = reader.readBytes(reader.readU16LE()); - const flags = reader.readU8(); - const large = (flags & 0x80) !== 0; - const compression = flags & 0x7f; - decodeAssert(compression === 0 || compression === 1, "invalid SSTable compression tag", reader.position - 1); - const lastKey = large ? undefined : reader.readBytes(reader.readU16LE()); - metadata.push({ offset, large, compression, firstKey, lastKey }); - } - reader.readU32LE(); - reader.assertEnd("trailing SSTable metadata bytes"); - for (let index = 0; index < metadata.length; index += 1) { - const current = metadata[index]; - decodeAssert(current.firstKey.length > 0, "empty SSTable metadata key"); - if (index > 0) { - decodeAssert(current.offset > metadata[index - 1].offset, "SSTable block offsets are not strictly increasing"); - decodeAssert(compareBytes(metadata[index - 1].firstKey, current.firstKey) < 0, "SSTable metadata keys are not strictly increasing"); - } - } - return metadata; -} -function decodeNormalBlock(bytes, metadata) { - decodeAssert(bytes.length >= 2, "normal SSTable block is too short"); - const count = new ByteReader(bytes, bytes.length - 2, 2).readU16LE(); - decodeAssert(count > 0, "normal SSTable block is empty"); - const offsetsLength = count * 2; - const dataEnd = bytes.length - offsetsLength - 2; - decodeAssert(dataEnd >= 0, "normal SSTable offsets exceed block length"); - const offsetsReader = new ByteReader(bytes, dataEnd, offsetsLength); - const offsets = []; - for (let index = 0; index < count; index += 1) { - offsets.push(offsetsReader.readU16LE()); - } - decodeAssert(offsets[0] === 0, "first normal SSTable offset is not zero"); - const entries = []; - for (let index = 0; index < count; index += 1) { - const start = offsets[index]; - const end = index + 1 < count ? offsets[index + 1] : dataEnd; - decodeAssert(start <= end && end <= dataEnd, "invalid normal SSTable entry range"); - const entry = bytes.subarray(start, end); - if (index === 0) { - entries.push({ key: metadata.firstKey, value: entry }); - continue; - } - const reader = new ByteReader(entry); - const common = reader.readU8(); - decodeAssert(common <= metadata.firstKey.length, "invalid SSTable key prefix length"); - const suffix = reader.readBytes(reader.readU16LE()); - const key = new Uint8Array(common + suffix.length); - key.set(metadata.firstKey.subarray(0, common)); - key.set(suffix, common); - entries.push({ key, value: reader.readRemaining() }); - } - validateEntryOrder(entries); - const last = entries[entries.length - 1]; - decodeAssert(metadata.lastKey !== undefined && bytesEqual(last.key, metadata.lastKey), "SSTable metadata last key mismatch"); - return entries; -} -function encodeStoredBlock(raw, large, firstKey, lastKey, compression) { - const lz4 = compression === "none" ? undefined : encodeLz4FrameRaw(raw); - const useLz4 = compression === "lz4" || (compression === "auto" && lz4.length <= raw.length); - const payload = useLz4 ? lz4 : raw; - const writer = new ByteWriter(payload.length + 4); - writer.writeBytes(payload); - writer.writeU32LE(xxhash32(payload, LORO_XXHASH_SEED)); - return { - bytes: writer.toUint8Array(), - large, - compression: useLz4 ? 1 : 0, - firstKey, - lastKey, - }; -} -function encodeTable(blocks) { - encodeAssert(blocks.length > 0 && blocks.length <= MAX_BLOCK_COUNT, "invalid SSTable block count"); - let offset = 5; - const metadata = []; - for (const block of blocks) { - metadata.push({ - offset, - large: block.large, - compression: block.compression, - firstKey: block.firstKey, - lastKey: block.lastKey, - }); - offset += block.bytes.length; - encodeAssert(offset <= 4294967295, "SSTable exceeds u32 offsets"); - } - const metadataOffset = offset; - const metadataWriter = new ByteWriter(); - metadataWriter.writeU32LE(metadata.length); - for (const item of metadata) { - encodeAssert(item.firstKey.length <= 0xffff, "SSTable key exceeds u16 length"); - metadataWriter.writeU32LE(item.offset); - metadataWriter.writeU16LE(item.firstKey.length); - metadataWriter.writeBytes(item.firstKey); - metadataWriter.writeU8((item.large ? 0x80 : 0) | item.compression); - if (!item.large) { - encodeAssert(item.lastKey !== undefined, "normal SSTable block lacks last key"); - encodeAssert(item.lastKey.length <= 0xffff, "SSTable key exceeds u16 length"); - metadataWriter.writeU16LE(item.lastKey.length); - metadataWriter.writeBytes(item.lastKey); - } - } - const metadataWithoutChecksum = metadataWriter.toUint8Array(); - metadataWriter.writeU32LE(xxhash32(metadataWithoutChecksum.subarray(4), LORO_XXHASH_SEED)); - const writer = new ByteWriter(); - writer.writeBytes(SSTABLE_MAGIC); - writer.writeU8(SSTABLE_SCHEMA); - for (const block of blocks) { - writer.writeBytes(block.bytes); - } - writer.writeBytes(metadataWriter.toUint8Array()); - writer.writeU32LE(metadataOffset); - return writer.toUint8Array(); -} -function commonPrefixLength(a, b) { - const length = Math.min(255, a.length, b.length); - let index = 0; - while (index < length && a[index] === b[index]) { - index += 1; - } - return index; -} -function validateEntriesForEncoding(entries) { - for (let index = 0; index < entries.length; index += 1) { - const current = entries[index]; - if (current.key.length === 0) { - throw new LoroEncodeError("SSTable keys cannot be empty"); - } - if (current.key.length > 0xffff) { - throw new LoroEncodeError("SSTable key exceeds u16 length"); - } - if (index > 0 && compareBytes(entries[index - 1].key, current.key) >= 0) { - throw new LoroEncodeError("SSTable keys must be unique"); - } - } -} -function validateEntryOrder(entries) { - for (let index = 1; index < entries.length; index += 1) { - if (compareBytes(entries[index - 1].key, entries[index].key) >= 0) { - throw new LoroDecodeError("SSTable keys are not strictly increasing"); - } - } -} diff --git a/loro-js/src/codec/state-snapshot.js b/loro-js/src/codec/state-snapshot.js deleted file mode 100644 index 6e82072c4..000000000 --- a/loro-js/src/codec/state-snapshot.js +++ /dev/null @@ -1,731 +0,0 @@ -import { ByteReader, ByteWriter, bytesEqual, compareBytes } from "./bytes"; -import { containerTypeFromRawByte, containerTypeToRawByte, decodeContainerId, encodeContainerId, readPostcardOptionalContainerId, writePostcardOptionalContainerId, } from "./container-id"; -import { decodeAssert, encodeAssert } from "./errors"; -import { U64_MAX, readUleb128, readUlebNumber, writeUleb128 } from "./leb128"; -import { PostcardReader, PostcardWriter } from "./postcard"; -import { decodePositionArena, encodePositionArena } from "./position-arena"; -import { decodeBoolRle, decodeColumnarVecMaybeWrapped, decodeDeltaRleI32, decodeDeltaRleU32, decodeDeltaRleUsize, encodeBoolRle, encodeColumnarVec, encodeColumnarVecWrapped, encodeDeltaRleI32, encodeDeltaRleU32, encodeDeltaRleUsize, takeColumnarVec, } from "./serde-columnar"; -import { decodeSstable, encodeSstable, } from "./sstable"; -import { ContainerType, } from "./types"; -import { readPostcardValue, readPostcardValueMap, readPostcardValues, writePostcardValue, writePostcardValueMap, writePostcardValues, } from "./value"; -import { decodePostcardFrontiers, encodePostcardFrontiers } from "./version"; -const MAX_STATE_ITEMS = 10000000; -const EMPTY_STATE_SENTINEL = Uint8Array.of(0x45); -const FRONTIERS_KEY = Uint8Array.of(0x66, 0x72); -const textEncoder = new TextEncoder(); -export function decodeMapStateSnapshot(bytes) { - const reader = new PostcardReader(bytes); - const values = readPostcardValueMap(reader); - const deletedKeys = readStringVector(reader, "map deleted keys"); - const peers = readPeerTable(reader.input); - const keys = collectSortedMapKeysForDecode(values, deletedKeys); - const peerCount = BigInt(peers.length); - const metadata = []; - for (const key of keys) { - const peerIndex = readUleb128(reader.input, U64_MAX); - const lamport = readUleb128(reader.input, U64_MAX); - assertDecodedPeerIndex(peerIndex, peerCount, "map metadata"); - metadata.push({ key, peerIndex, lamport }); - } - reader.assertEnd(); - sortByUtf8Key(values, (entry) => entry[0]); - sortByUtf8Key(deletedKeys, (key) => key); - return { kind: ContainerType.Map, values, deletedKeys, peers, metadata }; -} -export function encodeMapStateSnapshot(state) { - const values = [...state.values]; - sortByUtf8Key(values, (entry) => entry[0]); - const deletedKeys = [...state.deletedKeys]; - sortByUtf8Key(deletedKeys, (key) => key); - const keys = collectSortedMapKeysForEncode(values, deletedKeys); - const metadataByKey = new Map(); - for (const metadata of state.metadata) { - encodeAssert(!metadataByKey.has(metadata.key), `duplicate map metadata key ${metadata.key}`); - metadataByKey.set(metadata.key, metadata); - } - encodeAssert(metadataByKey.size === keys.length, "map metadata length mismatch"); - const output = new ByteWriter(); - const postcard = new PostcardWriter(output); - writePostcardValueMap(postcard, values); - postcard.writeArray(deletedKeys, (writer, key) => writer.writeString(key)); - writePeerTable(output, state.peers); - const peerCount = BigInt(state.peers.length); - for (const key of keys) { - const metadata = metadataByKey.get(key); - encodeAssert(metadata !== undefined, `missing map metadata for key ${key}`); - assertEncodedPeerIndex(metadata.peerIndex, peerCount, "map metadata"); - writeUleb128(output, metadata.peerIndex); - writeUleb128(output, metadata.lamport); - } - return output.toUint8Array(); -} -export function decodeListStateSnapshot(bytes) { - const reader = new PostcardReader(bytes); - const values = readPostcardValues(reader); - const peers = readPeerTable(reader.input); - const columns = decodeColumnarVecMaybeWrapped(reader.input.readRemaining()); - decodeAssert(columns.length === 3, "list state must have three ID columns"); - const peerIndices = decodeDeltaRleUsize(columns[0]); - const counters = decodeDeltaRleI32(columns[1]); - const lamportSubs = decodeDeltaRleI32(columns[2]); - decodeAssert(peerIndices.length === values.length && - counters.length === values.length && - lamportSubs.length === values.length, "list state ID length mismatch"); - const peerCount = BigInt(peers.length); - const ids = peerIndices.map((peerIndex, index) => { - assertDecodedPeerIndex(peerIndex, peerCount, "list state ID"); - return { - peerIndex, - counter: counters[index], - lamportSub: lamportSubs[index], - }; - }); - return { kind: ContainerType.List, values, peers, ids }; -} -export function encodeListStateSnapshot(state) { - encodeAssert(state.ids.length === state.values.length, "list state ID length mismatch"); - const peerCount = BigInt(state.peers.length); - for (const id of state.ids) { - assertEncodedPeerIndex(id.peerIndex, peerCount, "list state ID"); - } - const ids = state.ids; - const peerIndices = new Array(ids.length); - const counters = new Array(ids.length); - const lamportSubs = new Array(ids.length); - for (let index = 0; index < ids.length; index += 1) { - const id = ids[index]; - peerIndices[index] = id.peerIndex; - counters[index] = id.counter; - lamportSubs[index] = id.lamportSub; - } - const output = new ByteWriter(); - const postcard = new PostcardWriter(output); - writePostcardValues(postcard, state.values); - writePeerTable(output, state.peers); - output.writeBytes(encodeColumnarVecWrapped([ - encodeDeltaRleUsize(peerIndices), - encodeDeltaRleI32(counters), - encodeDeltaRleI32(lamportSubs), - ])); - return output.toUint8Array(); -} -export function decodeTextStateSnapshot(bytes) { - const reader = new PostcardReader(bytes); - const text = reader.readString(); - const peers = readPeerTable(reader.input); - decodeAssert(reader.readUsize() === 3, "text state must have three encoded fields"); - const columns = readColumnarVecFrom(reader.input); - decodeAssert(columns.length === 4, "text state must have four span columns"); - const peerIndices = decodeDeltaRleUsize(columns[0]); - const counters = decodeDeltaRleI32(columns[1]); - const lamportSubs = decodeDeltaRleI32(columns[2]); - const lengths = decodeDeltaRleI32(columns[3]); - decodeAssert(peerIndices.length === lengths.length && - counters.length === lengths.length && - lamportSubs.length === lengths.length, "text state span column length mismatch"); - const peerCount = BigInt(peers.length); - const spans = lengths.map((length, index) => { - const peerIndex = peerIndices[index]; - assertDecodedPeerIndex(peerIndex, peerCount, "text state span"); - return { - peerIndex, - counter: counters[index], - lamportSub: lamportSubs[index], - length, - }; - }); - const keys = readStringVector(reader, "text mark keys"); - const markCount = reader.readUsize(); - decodeAssert(markCount <= MAX_STATE_ITEMS, "too many text marks"); - const marks = []; - for (let index = 0; index < markCount; index += 1) { - decodeAssert(reader.readUsize() === 3, "text mark must have three encoded fields"); - const keyIndex = reader.readUsize(); - const value = readPostcardValue(reader); - const info = reader.readU8(); - decodeAssert(keyIndex < keys.length, "text mark key index out of range"); - marks.push({ keyIndex, value, info }); - } - reader.assertEnd(); - validateDecodedText(text, spans, marks); - return { kind: ContainerType.Text, text, peers, spans, keys, marks }; -} -export function encodeTextStateSnapshot(state) { - validateEncodedText(state); - const spans = state.spans; - const peerIndices = new Array(spans.length); - const counters = new Array(spans.length); - const lamportSubs = new Array(spans.length); - const lengths = new Array(spans.length); - for (let index = 0; index < spans.length; index += 1) { - const span = spans[index]; - peerIndices[index] = span.peerIndex; - counters[index] = span.counter; - lamportSubs[index] = span.lamportSub; - lengths[index] = span.length; - } - const output = new ByteWriter(); - const postcard = new PostcardWriter(output); - postcard.writeString(state.text); - writePeerTable(output, state.peers); - postcard.writeUsize(3); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(peerIndices), - encodeDeltaRleI32(counters), - encodeDeltaRleI32(lamportSubs), - encodeDeltaRleI32(lengths), - ])); - postcard.writeArray(state.keys, (writer, key) => writer.writeString(key)); - postcard.writeUsize(state.marks.length); - for (const mark of state.marks) { - postcard.writeUsize(3); - postcard.writeUsize(mark.keyIndex); - writePostcardValue(postcard, mark.value); - postcard.writeU8(mark.info); - } - return output.toUint8Array(); -} -export function decodeTreeStateSnapshot(bytes) { - const input = new ByteReader(bytes); - const peers = readPeerTable(input); - const reader = new PostcardReader(input); - decodeAssert(reader.readUsize() === 4, "tree state must have four encoded fields"); - const idColumns = readColumnarVecFrom(input); - decodeAssert(idColumns.length === 2, "tree state must have two node ID columns"); - const nodePeerIndices = decodeDeltaRleUsize(idColumns[0]); - const nodeCounters = decodeDeltaRleI32(idColumns[1]); - decodeAssert(nodePeerIndices.length === nodeCounters.length, "tree state node ID column length mismatch"); - const nodeColumns = readColumnarVecFrom(input); - decodeAssert(nodeColumns.length === 5, "tree state must have five node columns"); - const parentIndices = decodeDeltaRleUsize(nodeColumns[0]); - const lastSetPeerIndices = decodeDeltaRleUsize(nodeColumns[1]); - const lastSetCounters = decodeDeltaRleI32(nodeColumns[2]); - const lastSetLamportSubs = decodeDeltaRleI32(nodeColumns[3]); - const fractionalIndexIndices = decodePostcardUsizeVector(nodeColumns[4]); - const length = nodePeerIndices.length; - decodeAssert(parentIndices.length === length && - lastSetPeerIndices.length === length && - lastSetCounters.length === length && - lastSetLamportSubs.length === length && - fractionalIndexIndices.length === length, "tree state node column length mismatch"); - const positions = decodePositionArena(reader.readBytes()); - const reserved = reader.readBytes(); - reader.assertEnd(); - const peerCount = BigInt(peers.length); - const parentIndexBound = BigInt(length + 1); - const nodes = nodePeerIndices.map((peerIndex, index) => { - const lastSetPeerIndex = lastSetPeerIndices[index]; - const parentIndexPlusTwo = parentIndices[index]; - const fractionalIndexIndex = fractionalIndexIndices[index]; - assertDecodedPeerIndex(peerIndex, peerCount, "tree node ID"); - assertDecodedPeerIndex(lastSetPeerIndex, peerCount, "tree last-set ID"); - decodeAssert(parentIndexPlusTwo <= parentIndexBound, "tree parent index out of range"); - decodeAssert(fractionalIndexIndex < positions.length, "tree fractional index out of range"); - return { - peerIndex, - counter: nodeCounters[index], - parentIndexPlusTwo, - lastSetPeerIndex, - lastSetCounter: lastSetCounters[index], - lastSetLamportSub: lastSetLamportSubs[index], - fractionalIndexIndex, - }; - }); - return { kind: ContainerType.Tree, peers, nodes, positions, reserved }; -} -export function encodeTreeStateSnapshot(state) { - validateEncodedTree(state); - const nodes = state.nodes; - const nodePeerIndices = new Array(nodes.length); - const nodeCounters = new Array(nodes.length); - const parentIndices = new Array(nodes.length); - const lastSetPeerIndices = new Array(nodes.length); - const lastSetCounters = new Array(nodes.length); - const lastSetLamportSubs = new Array(nodes.length); - const fractionalIndexIndices = new Array(nodes.length); - for (let index = 0; index < nodes.length; index += 1) { - const node = nodes[index]; - nodePeerIndices[index] = node.peerIndex; - nodeCounters[index] = node.counter; - parentIndices[index] = node.parentIndexPlusTwo; - lastSetPeerIndices[index] = node.lastSetPeerIndex; - lastSetCounters[index] = node.lastSetCounter; - lastSetLamportSubs[index] = node.lastSetLamportSub; - fractionalIndexIndices[index] = node.fractionalIndexIndex; - } - const output = new ByteWriter(); - writePeerTable(output, state.peers); - const postcard = new PostcardWriter(output); - postcard.writeUsize(4); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(nodePeerIndices), - encodeDeltaRleI32(nodeCounters), - ])); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(parentIndices), - encodeDeltaRleUsize(lastSetPeerIndices), - encodeDeltaRleI32(lastSetCounters), - encodeDeltaRleI32(lastSetLamportSubs), - encodePostcardUsizeVector(fractionalIndexIndices), - ])); - postcard.writeBytes(encodePositionArena(state.positions, { encodeEmpty: true })); - postcard.writeBytes(state.reserved); - return output.toUint8Array(); -} -export function decodeMovableListStateSnapshot(bytes) { - const reader = new PostcardReader(bytes); - const values = readPostcardValues(reader); - const peers = readPeerTable(reader.input); - decodeAssert(reader.readUsize() === 4, "movable-list state must have four encoded fields"); - const itemColumns = readColumnarVecFrom(reader.input); - decodeAssert(itemColumns.length === 3, "movable-list state item column count"); - const invisibleListItems = decodeDeltaRleUsize(itemColumns[0]); - const positionIdEqualsElementId = decodeBoolRle(itemColumns[1]); - const elementIdEqualsLastSetId = decodeBoolRle(itemColumns[2]); - decodeAssert(invisibleListItems.length === positionIdEqualsElementId.length && - invisibleListItems.length === elementIdEqualsLastSetId.length, "movable-list state item column length mismatch"); - const listIdColumns = readColumnarVecFrom(reader.input); - decodeAssert(listIdColumns.length === 3, "movable-list list ID column count"); - const listPeerIndices = decodeDeltaRleUsize(listIdColumns[0]); - const listCounters = decodeDeltaRleI32(listIdColumns[1]); - const listLamportSubs = decodeDeltaRleI32(listIdColumns[2]); - decodeAssert(listPeerIndices.length === listCounters.length && - listPeerIndices.length === listLamportSubs.length, "movable-list list ID column length mismatch"); - const elementIdColumns = readColumnarVecFrom(reader.input); - decodeAssert(elementIdColumns.length === 2, "movable-list element ID column count"); - const elementPeerIndices = decodeDeltaRleUsize(elementIdColumns[0]); - const elementLamports = decodeDeltaRleU32(elementIdColumns[1]); - decodeAssert(elementPeerIndices.length === elementLamports.length, "movable-list element ID column length mismatch"); - const lastSetIdColumns = readColumnarVecFrom(reader.input); - decodeAssert(lastSetIdColumns.length === 2, "movable-list last-set ID column count"); - const lastSetPeerIndices = decodeDeltaRleUsize(lastSetIdColumns[0]); - const lastSetLamports = decodeDeltaRleU32(lastSetIdColumns[1]); - decodeAssert(lastSetPeerIndices.length === lastSetLamports.length, "movable-list last-set ID column length mismatch"); - reader.assertEnd(); - const items = invisibleListItems.map((invisibleListItemsForItem, index) => ({ - invisibleListItems: invisibleListItemsForItem, - positionIdEqualsElementId: positionIdEqualsElementId[index], - elementIdEqualsLastSetId: elementIdEqualsLastSetId[index], - })); - const listItemIds = listPeerIndices.map((peerIndex, index) => ({ - peerIndex, - counter: listCounters[index], - lamportSub: listLamportSubs[index], - })); - const elementIds = elementPeerIndices.map((peerIndex, index) => ({ - peerIndex, - lamport: elementLamports[index], - })); - const lastSetIds = lastSetPeerIndices.map((peerIndex, index) => ({ - peerIndex, - lamport: lastSetLamports[index], - })); - validateDecodedMovableList(values.length, BigInt(peers.length), items, listItemIds, elementIds, lastSetIds); - return { - kind: ContainerType.MovableList, - values, - peers, - items, - listItemIds, - elementIds, - lastSetIds, - }; -} -export function encodeMovableListStateSnapshot(state) { - validateEncodedMovableList(state); - const items = state.items; - const invisibleListItems = new Array(items.length); - const positionIdEqualsElementId = new Array(items.length); - const elementIdEqualsLastSetId = new Array(items.length); - for (let index = 0; index < items.length; index += 1) { - const item = items[index]; - invisibleListItems[index] = item.invisibleListItems; - positionIdEqualsElementId[index] = item.positionIdEqualsElementId; - elementIdEqualsLastSetId[index] = item.elementIdEqualsLastSetId; - } - const listItemIds = state.listItemIds; - const listPeerIndices = new Array(listItemIds.length); - const listCounters = new Array(listItemIds.length); - const listLamportSubs = new Array(listItemIds.length); - for (let index = 0; index < listItemIds.length; index += 1) { - const id = listItemIds[index]; - listPeerIndices[index] = id.peerIndex; - listCounters[index] = id.counter; - listLamportSubs[index] = id.lamportSub; - } - const elementIds = state.elementIds; - const elementPeerIndices = new Array(elementIds.length); - const elementLamports = new Array(elementIds.length); - for (let index = 0; index < elementIds.length; index += 1) { - const id = elementIds[index]; - elementPeerIndices[index] = id.peerIndex; - elementLamports[index] = id.lamport; - } - const lastSetIds = state.lastSetIds; - const lastSetPeerIndices = new Array(lastSetIds.length); - const lastSetLamports = new Array(lastSetIds.length); - for (let index = 0; index < lastSetIds.length; index += 1) { - const id = lastSetIds[index]; - lastSetPeerIndices[index] = id.peerIndex; - lastSetLamports[index] = id.lamport; - } - const output = new ByteWriter(); - const postcard = new PostcardWriter(output); - writePostcardValues(postcard, state.values); - writePeerTable(output, state.peers); - postcard.writeUsize(4); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(invisibleListItems), - encodeBoolRle(positionIdEqualsElementId), - encodeBoolRle(elementIdEqualsLastSetId), - ])); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(listPeerIndices), - encodeDeltaRleI32(listCounters), - encodeDeltaRleI32(listLamportSubs), - ])); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(elementPeerIndices), - encodeDeltaRleU32(elementLamports), - ])); - output.writeBytes(encodeColumnarVec([ - encodeDeltaRleUsize(lastSetPeerIndices), - encodeDeltaRleU32(lastSetLamports), - ])); - return output.toUint8Array(); -} -export function decodeCounterStateSnapshot(bytes) { - decodeAssert(bytes.length === 8, "counter state must contain exactly eight bytes"); - return { kind: ContainerType.Counter, bits: new ByteReader(bytes).readU64LE() }; -} -export function encodeCounterStateSnapshot(state) { - const output = new ByteWriter(8); - output.writeU64LE(state.bits); - return output.toUint8Array(); -} -export function decodeContainerStateSnapshot(containerType, bytes) { - switch (containerType) { - case ContainerType.Map: - return decodeMapStateSnapshot(bytes); - case ContainerType.List: - return decodeListStateSnapshot(bytes); - case ContainerType.Text: - return decodeTextStateSnapshot(bytes); - case ContainerType.Tree: - return decodeTreeStateSnapshot(bytes); - case ContainerType.MovableList: - return decodeMovableListStateSnapshot(bytes); - case ContainerType.Counter: - return decodeCounterStateSnapshot(bytes); - default: - return { kind: containerType, payload: bytes.slice() }; - } -} -export function encodeContainerStateSnapshot(state) { - if (typeof state.kind !== "string") { - return state.payload.slice(); - } - switch (state.kind) { - case ContainerType.Map: - return encodeMapStateSnapshot(state); - case ContainerType.List: - return encodeListStateSnapshot(state); - case ContainerType.Text: - return encodeTextStateSnapshot(state); - case ContainerType.Tree: - return encodeTreeStateSnapshot(state); - case ContainerType.MovableList: - return encodeMovableListStateSnapshot(state); - case ContainerType.Counter: - return encodeCounterStateSnapshot(state); - } -} -export function decodeContainerStateWrapper(bytes) { - const input = new ByteReader(bytes); - decodeAssert(input.remaining > 0, "container state wrapper is empty"); - const containerType = containerTypeFromRawByte(input.readU8()); - const depth = readUleb128(input, U64_MAX); - const postcard = new PostcardReader(input); - const parent = readPostcardOptionalContainerId(postcard); - const state = decodeContainerStateSnapshot(containerType, input.readRemaining()); - return { containerType, depth, parent, state }; -} -export function encodeContainerStateWrapper(wrapper) { - encodeAssert(sameContainerType(wrapper.containerType, wrapper.state.kind), "container wrapper type does not match its state"); - const output = new ByteWriter(); - output.writeU8(containerTypeToRawByte(wrapper.containerType)); - writeUleb128(output, wrapper.depth); - writePostcardOptionalContainerId(new PostcardWriter(output), wrapper.parent); - output.writeBytes(encodeContainerStateSnapshot(wrapper.state)); - return output.toUint8Array(); -} -export function decodeStateSnapshotStore(bytes, options) { - if (bytes.length === 0) { - return { kind: "absent" }; - } - if (bytesEqual(bytes, EMPTY_STATE_SENTINEL)) { - return { kind: "empty" }; - } - const containers = []; - let frontiers; - for (const entry of decodeSstable(bytes, options)) { - if (bytesEqual(entry.key, FRONTIERS_KEY)) { - decodeAssert(frontiers === undefined, "duplicate state frontiers entry"); - frontiers = decodePostcardFrontiers(entry.value); - continue; - } - const id = decodeContainerId(entry.key); - const wrapper = decodeContainerStateWrapper(entry.value); - decodeAssert(sameContainerType(id.containerType, wrapper.containerType), "state container key and wrapper types differ"); - containers.push({ id, wrapper }); - } - return { kind: "sstable", frontiers, containers }; -} -export function encodeStateSnapshotStore(store, options) { - if (store.kind === "absent") { - return new Uint8Array(); - } - if (store.kind === "empty") { - return EMPTY_STATE_SENTINEL.slice(); - } - const entries = store.containers.map(({ id, wrapper }) => { - encodeAssert(sameContainerType(id.containerType, wrapper.containerType), "state container key and wrapper types differ"); - return { key: encodeContainerId(id), value: encodeContainerStateWrapper(wrapper) }; - }); - if (store.frontiers !== undefined) { - entries.push({ key: FRONTIERS_KEY, value: encodePostcardFrontiers(store.frontiers) }); - } - encodeAssert(entries.length > 0, "an SSTable state store must not be empty"); - return encodeSstable(entries, options); -} -function readPeerTable(reader) { - const length = readUlebNumber(reader, MAX_STATE_ITEMS); - decodeAssert(length <= Math.floor(reader.remaining / 8), "state peer table is truncated"); - const peers = []; - for (let index = 0; index < length; index += 1) { - peers.push(reader.readU64LE()); - } - return peers; -} -function writePeerTable(writer, peers) { - encodeAssert(peers.length <= MAX_STATE_ITEMS, "state peer table is too large"); - writeUleb128(writer, peers.length); - for (const peer of peers) { - writer.writeU64LE(peer); - } -} -function readColumnarVecFrom(reader) { - const remaining = reader.bytes.subarray(reader.position, reader.position + reader.remaining); - const [columns, rest] = takeColumnarVec(remaining); - reader.skip(remaining.length - rest.length); - return columns; -} -function readStringVector(reader, label) { - const length = reader.readUsize(); - decodeAssert(length <= MAX_STATE_ITEMS, `${label} is too large`); - decodeAssert(length <= reader.input.remaining, `${label} exceeds remaining input`); - const values = []; - for (let index = 0; index < length; index += 1) { - values.push(reader.readString()); - } - return values; -} -function decodePostcardUsizeVector(bytes) { - const reader = new PostcardReader(bytes); - const length = reader.readUsize(); - decodeAssert(length <= MAX_STATE_ITEMS, "postcard usize vector is too large"); - decodeAssert(length <= reader.input.remaining, "postcard usize vector is truncated"); - const values = []; - for (let index = 0; index < length; index += 1) { - values.push(reader.readUsize()); - } - reader.assertEnd(); - return values; -} -function encodePostcardUsizeVector(values) { - const writer = new PostcardWriter(); - writer.writeUsize(values.length); - for (const value of values) { - writer.writeUsize(value); - } - return writer.toUint8Array(); -} -function collectSortedMapKeysForDecode(values, deletedKeys) { - const keys = mergeMapKeys(values, deletedKeys); - for (let index = 1; index < keys.length; index += 1) { - decodeAssert(keys[index] !== keys[index - 1], "duplicate map state key"); - } - return keys; -} -function collectSortedMapKeysForEncode(values, deletedKeys) { - const keys = mergeMapKeys(values, deletedKeys); - for (let index = 1; index < keys.length; index += 1) { - encodeAssert(keys[index] !== keys[index - 1], "duplicate map state key"); - } - return keys; -} -function mergeMapKeys(values, deletedKeys) { - const keys = new Array(values.length + deletedKeys.length); - let offset = 0; - for (const [key] of values) { - keys[offset] = key; - offset += 1; - } - for (const key of deletedKeys) { - keys[offset] = key; - offset += 1; - } - sortByUtf8Key(keys, (key) => key); - return keys; -} -/** - * Sorts `items` in place by the UTF-8 byte order of their keys (interop - * requires UTF-8 byte order, not UTF-16 code unit order). Each key is encoded - * exactly once instead of once per comparison. - */ -function sortByUtf8Key(items, keyOf) { - const decorated = items.map((item) => ({ - bytes: textEncoder.encode(keyOf(item)), - item, - })); - decorated.sort((left, right) => compareBytes(left.bytes, right.bytes)); - for (let index = 0; index < decorated.length; index += 1) { - items[index] = decorated[index].item; - } -} -function assertDecodedPeerIndex(index, peerCount, label) { - decodeAssert(index < peerCount, `${label} peer index out of range`); -} -function assertEncodedPeerIndex(index, peerCount, label) { - encodeAssert(index >= 0n && index < peerCount, `${label} peer index out of range`); -} -function validateDecodedText(text, spans, marks) { - let markCount = 0; - let textLength = 0; - for (const span of spans) { - if (span.length === 0) { - markCount += 1; - } - else if (span.length > 0) { - textLength += span.length; - } - } - decodeAssert(markCount === marks.length, "text state mark count mismatch"); - decodeAssert(unicodeScalarLength(text) === textLength, "text state Unicode length mismatch"); -} -function validateEncodedText(state) { - let markCount = 0; - let textLength = 0; - const peerCount = BigInt(state.peers.length); - for (const span of state.spans) { - assertEncodedPeerIndex(span.peerIndex, peerCount, "text state span"); - if (span.length === 0) { - markCount += 1; - } - else if (span.length > 0) { - textLength += span.length; - } - } - encodeAssert(markCount === state.marks.length, "text state mark count mismatch"); - encodeAssert(unicodeScalarLength(state.text) === textLength, "text state Unicode length mismatch"); - for (const mark of state.marks) { - encodeAssert(mark.keyIndex >= 0 && mark.keyIndex < state.keys.length, "text mark key index out of range"); - } -} -function unicodeScalarLength(value) { - let length = 0; - for (let index = 0; index < value.length; index += 1) { - const codeUnit = value.charCodeAt(index); - if (codeUnit >= 0xd800 && - codeUnit <= 0xdbff && - index + 1 < value.length && - value.charCodeAt(index + 1) >= 0xdc00 && - value.charCodeAt(index + 1) <= 0xdfff) { - index += 1; - } - length += 1; - } - return length; -} -function validateEncodedTree(state) { - const peerCount = BigInt(state.peers.length); - const parentIndexBound = BigInt(state.nodes.length + 1); - for (const node of state.nodes) { - assertEncodedPeerIndex(node.peerIndex, peerCount, "tree node ID"); - assertEncodedPeerIndex(node.lastSetPeerIndex, peerCount, "tree last-set ID"); - encodeAssert(node.parentIndexPlusTwo >= 0n && - node.parentIndexPlusTwo <= parentIndexBound, "tree parent index out of range"); - encodeAssert(Number.isSafeInteger(node.fractionalIndexIndex) && - node.fractionalIndexIndex >= 0 && - node.fractionalIndexIndex < state.positions.length, "tree fractional index out of range"); - } -} -function validateDecodedMovableList(valueCount, peerCount, items, listItemIds, elementIds, lastSetIds) { - const visibleCount = items.length === 0 ? 0 : items.length - 1; - decodeAssert(valueCount === visibleCount, "movable-list visible value count mismatch"); - // Decoded usize values are non-negative, so the Number running total stays - // exact whenever it could equal an actual array length. - let expectedListIds = visibleCount; - for (const item of items) { - expectedListIds += Number(item.invisibleListItems); - } - decodeAssert(expectedListIds === listItemIds.length, "movable-list list ID count mismatch"); - let expectedElementIds = 0; - let expectedLastSetIds = 0; - for (let index = 1; index < items.length; index += 1) { - if (!items[index].positionIdEqualsElementId) { - expectedElementIds += 1; - } - if (!items[index].elementIdEqualsLastSetId) { - expectedLastSetIds += 1; - } - } - decodeAssert(elementIds.length === expectedElementIds, "movable-list element ID count mismatch"); - decodeAssert(lastSetIds.length === expectedLastSetIds, "movable-list last-set ID count mismatch"); - for (const id of listItemIds) { - assertDecodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); - } - for (const id of elementIds) { - assertDecodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); - } - for (const id of lastSetIds) { - assertDecodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); - } -} -function validateEncodedMovableList(state) { - const visibleCount = state.items.length === 0 ? 0 : state.items.length - 1; - encodeAssert(state.values.length === visibleCount, "movable-list visible value count mismatch"); - // Items were asserted non-negative, so the Number running total stays exact - // whenever it could equal an actual array length. - let expectedListIds = visibleCount; - for (const item of state.items) { - encodeAssert(item.invisibleListItems >= 0n, "movable-list invisible count is negative"); - expectedListIds += Number(item.invisibleListItems); - } - encodeAssert(expectedListIds === state.listItemIds.length, "movable-list list ID count mismatch"); - let expectedElementIds = 0; - let expectedLastSetIds = 0; - for (let index = 1; index < state.items.length; index += 1) { - if (!state.items[index].positionIdEqualsElementId) { - expectedElementIds += 1; - } - if (!state.items[index].elementIdEqualsLastSetId) { - expectedLastSetIds += 1; - } - } - encodeAssert(state.elementIds.length === expectedElementIds, "movable-list element ID count mismatch"); - encodeAssert(state.lastSetIds.length === expectedLastSetIds, "movable-list last-set ID count mismatch"); - const peerCount = BigInt(state.peers.length); - for (const id of state.listItemIds) { - assertEncodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); - } - for (const id of state.elementIds) { - assertEncodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); - } - for (const id of state.lastSetIds) { - assertEncodedPeerIndex(id.peerIndex, peerCount, "movable-list ID"); - } -} -function sameContainerType(left, right) { - return containerTypeToRawByte(left) === containerTypeToRawByte(right); -} diff --git a/loro-js/src/codec/types.js b/loro-js/src/codec/types.js deleted file mode 100644 index fce266bb7..000000000 --- a/loro-js/src/codec/types.js +++ /dev/null @@ -1,8 +0,0 @@ -export const ContainerType = { - Map: "map", - List: "list", - Text: "text", - Tree: "tree", - MovableList: "movable-list", - Counter: "counter", -}; diff --git a/loro-js/src/codec/value.js b/loro-js/src/codec/value.js deleted file mode 100644 index 6ca34e7ff..000000000 --- a/loro-js/src/codec/value.js +++ /dev/null @@ -1,156 +0,0 @@ -import { readPostcardContainerId, writePostcardContainerId } from "./container-id"; -import { decodeAssert } from "./errors"; -import { PostcardReader, PostcardWriter } from "./postcard"; -const MAX_VALUE_DEPTH = 1024; -export function readPostcardValue(reader, depth = 0) { - decodeAssert(depth <= MAX_VALUE_DEPTH, "postcard LoroValue is too deep", reader.input.position); - const tag = reader.readU32(); - switch (tag) { - case 0: - return { type: "null" }; - case 1: - return { type: "bool", value: reader.readBool() }; - case 2: - return { type: "double", value: reader.readF64() }; - case 3: - return { type: "i64", value: reader.readI64() }; - case 4: - return { type: "string", value: reader.readString() }; - case 5: { - const length = reader.readUsize(); - decodeAssert(length <= reader.input.remaining, "postcard LoroValue list length exceeds remaining input", reader.input.position); - const value = []; - for (let index = 0; index < length; index += 1) { - value.push(readPostcardValue(reader, depth + 1)); - } - return { type: "list", value }; - } - case 6: { - const length = reader.readUsize(); - decodeAssert(length <= Math.floor(reader.input.remaining / 2), "postcard LoroValue map length exceeds remaining input", reader.input.position); - const value = []; - for (let index = 0; index < length; index += 1) { - value.push([reader.readString(), readPostcardValue(reader, depth + 1)]); - } - return { type: "map", value }; - } - case 7: - return { type: "container", value: readPostcardContainerId(reader) }; - case 8: - return { type: "binary", value: reader.readBytes() }; - default: - decodeAssert(false, "invalid postcard LoroValue discriminant", reader.input.position); - } -} -export function writePostcardValue(writer, value, depth = 0) { - if (depth > MAX_VALUE_DEPTH) { - throw new RangeError("postcard LoroValue is too deep"); - } - switch (value.type) { - case "null": - writer.writeU32(0); - return; - case "bool": - writer.writeU32(1); - writer.writeBool(value.value); - return; - case "double": - writer.writeU32(2); - writer.writeF64(value.value); - return; - case "i64": - writer.writeU32(3); - writer.writeI64(value.value); - return; - case "string": - writer.writeU32(4); - writer.writeString(value.value); - return; - case "list": - writer.writeU32(5); - writer.writeUsize(value.value.length); - for (const item of value.value) { - writePostcardValue(writer, item, depth + 1); - } - return; - case "map": - writer.writeU32(6); - writer.writeUsize(value.value.length); - for (const [key, item] of value.value) { - writer.writeString(key); - writePostcardValue(writer, item, depth + 1); - } - return; - case "container": - writer.writeU32(7); - writePostcardContainerId(writer, value.value); - return; - case "binary": - writer.writeU32(8); - writer.writeBytes(value.value); - } -} -export function decodePostcardValue(bytes) { - const reader = new PostcardReader(bytes); - const value = readPostcardValue(reader); - reader.assertEnd(); - return value; -} -export function encodePostcardValue(value) { - const writer = new PostcardWriter(); - writePostcardValue(writer, value); - return writer.toUint8Array(); -} -export function readPostcardValues(reader) { - const length = reader.readUsize(); - decodeAssert(length <= reader.input.remaining, "postcard LoroValue vector length exceeds remaining input", reader.input.position); - const values = []; - for (let index = 0; index < length; index += 1) { - values.push(readPostcardValue(reader)); - } - return values; -} -export function writePostcardValues(writer, values) { - writer.writeUsize(values.length); - for (const value of values) { - writePostcardValue(writer, value); - } -} -export function decodePostcardValues(bytes) { - const reader = new PostcardReader(bytes); - const values = readPostcardValues(reader); - reader.assertEnd(); - return values; -} -export function encodePostcardValues(values) { - const writer = new PostcardWriter(); - writePostcardValues(writer, values); - return writer.toUint8Array(); -} -export function readPostcardValueMap(reader) { - const length = reader.readUsize(); - decodeAssert(length <= Math.floor(reader.input.remaining / 2), "postcard LoroValue map length exceeds remaining input", reader.input.position); - const entries = []; - for (let index = 0; index < length; index += 1) { - entries.push([reader.readString(), readPostcardValue(reader)]); - } - return entries; -} -export function writePostcardValueMap(writer, entries) { - writer.writeUsize(entries.length); - for (const [key, value] of entries) { - writer.writeString(key); - writePostcardValue(writer, value); - } -} -export function decodePostcardValueMap(bytes) { - const reader = new PostcardReader(bytes); - const entries = readPostcardValueMap(reader); - reader.assertEnd(); - return entries; -} -export function encodePostcardValueMap(entries) { - const writer = new PostcardWriter(); - writePostcardValueMap(writer, entries); - return writer.toUint8Array(); -} diff --git a/loro-js/src/codec/version.js b/loro-js/src/codec/version.js deleted file mode 100644 index fef00a5c5..000000000 --- a/loro-js/src/codec/version.js +++ /dev/null @@ -1,60 +0,0 @@ -import { decodeAssert } from "./errors"; -import { assertId, readPostcardId, writePostcardId } from "./id"; -import { PostcardReader, PostcardWriter } from "./postcard"; -export function readPostcardVersionVector(reader) { - const length = reader.readUsize(); - decodeAssert(length <= Math.floor(reader.input.remaining / 2), "postcard version vector length exceeds remaining input", reader.input.position); - const version = []; - for (let index = 0; index < length; index += 1) { - version.push(readPostcardId(reader)); - } - return version; -} -export function writePostcardVersionVector(writer, version) { - writer.writeUsize(version.length); - for (const id of version) { - writePostcardId(writer, id); - } -} -export function decodePostcardVersionVector(bytes) { - const reader = new PostcardReader(bytes); - const version = readPostcardVersionVector(reader); - reader.assertEnd(); - return version; -} -export function encodePostcardVersionVector(version) { - const writer = new PostcardWriter(); - writePostcardVersionVector(writer, version); - return writer.toUint8Array(); -} -export function readPostcardFrontiers(reader) { - return readPostcardVersionVector(reader); -} -export function writePostcardFrontiers(writer, frontiers) { - const sorted = [...frontiers]; - for (const id of sorted) { - assertId(id); - } - sorted.sort(compareIds); - writePostcardVersionVector(writer, sorted); -} -export function decodePostcardFrontiers(bytes) { - const reader = new PostcardReader(bytes); - const frontiers = readPostcardFrontiers(reader); - reader.assertEnd(); - return frontiers; -} -export function encodePostcardFrontiers(frontiers) { - const writer = new PostcardWriter(); - writePostcardFrontiers(writer, frontiers); - return writer.toUint8Array(); -} -function compareIds(left, right) { - if (left.peer < right.peer) { - return -1; - } - if (left.peer > right.peer) { - return 1; - } - return left.counter - right.counter; -} diff --git a/loro-js/src/codec/xxhash32.js b/loro-js/src/codec/xxhash32.js deleted file mode 100644 index 53d3738b1..000000000 --- a/loro-js/src/codec/xxhash32.js +++ /dev/null @@ -1,68 +0,0 @@ -export const LORO_XXHASH_SEED = 1330794316; -const PRIME32_1 = 2654435761; -const PRIME32_2 = 2246822519; -const PRIME32_3 = 3266489917; -const PRIME32_4 = 668265263; -const PRIME32_5 = 374761393; -function rotateLeft(value, count) { - return ((value << count) | (value >>> (32 - count))) >>> 0; -} -function round(accumulator, input) { - let result = (accumulator + Math.imul(input, PRIME32_2)) >>> 0; - result = rotateLeft(result, 13); - return Math.imul(result, PRIME32_1) >>> 0; -} -function readU32LE(bytes, offset) { - return ((bytes[offset] | - (bytes[offset + 1] << 8) | - (bytes[offset + 2] << 16) | - (bytes[offset + 3] << 24)) >>> - 0); -} -export function xxhash32(bytes, seed = 0) { - let offset = 0; - let hash; - if (bytes.length >= 16) { - let v1 = (seed + PRIME32_1 + PRIME32_2) >>> 0; - let v2 = (seed + PRIME32_2) >>> 0; - let v3 = seed >>> 0; - let v4 = (seed - PRIME32_1) >>> 0; - const limit = bytes.length - 16; - while (offset <= limit) { - v1 = round(v1, readU32LE(bytes, offset)); - offset += 4; - v2 = round(v2, readU32LE(bytes, offset)); - offset += 4; - v3 = round(v3, readU32LE(bytes, offset)); - offset += 4; - v4 = round(v4, readU32LE(bytes, offset)); - offset += 4; - } - hash = - (rotateLeft(v1, 1) + - rotateLeft(v2, 7) + - rotateLeft(v3, 12) + - rotateLeft(v4, 18)) >>> - 0; - } - else { - hash = (seed + PRIME32_5) >>> 0; - } - hash = (hash + bytes.length) >>> 0; - while (offset + 4 <= bytes.length) { - hash = (hash + Math.imul(readU32LE(bytes, offset), PRIME32_3)) >>> 0; - hash = Math.imul(rotateLeft(hash, 17), PRIME32_4) >>> 0; - offset += 4; - } - while (offset < bytes.length) { - hash = (hash + Math.imul(bytes[offset], PRIME32_5)) >>> 0; - hash = Math.imul(rotateLeft(hash, 11), PRIME32_1) >>> 0; - offset += 1; - } - hash ^= hash >>> 15; - hash = Math.imul(hash, PRIME32_2) >>> 0; - hash ^= hash >>> 13; - hash = Math.imul(hash, PRIME32_3) >>> 0; - hash ^= hash >>> 16; - return hash >>> 0; -} diff --git a/loro-js/src/index.js b/loro-js/src/index.js deleted file mode 100644 index 3e3b526b1..000000000 --- a/loro-js/src/index.js +++ /dev/null @@ -1 +0,0 @@ -export * from "./runtime/index"; diff --git a/loro-js/src/runtime/containers.js b/loro-js/src/runtime/containers.js deleted file mode 100644 index f676b8810..000000000 --- a/loro-js/src/runtime/containers.js +++ /dev/null @@ -1,2597 +0,0 @@ -import { bytesToHex } from "../codec/bytes"; -import { containerTypeFromHistoricalByte, containerTypeToHistoricalByte, } from "../codec/container-id"; -import { PostcardReader, PostcardWriter } from "../codec/postcard"; -import { formatContainerId, formatTreeId, parseContainerId } from "./ids"; -import { fractionalIndexBetween } from "./fractional-index"; -import { OrderedIndex } from "./ordered-index"; -import { SequenceIndex, SequenceSpan } from "./sequence-index"; -import { TextStyleIndex } from "./text-style-index"; -export class LoroContainer { - _doc; - _codecId; - _containerId; - _parentLink; - _attached; - free() { } - get id() { - return this._containerId ?? `cid:0@0:${this.kind()}`; - } - isAttached() { - return this._doc !== undefined; - } - getAttached() { - return (this.isAttached() ? this : this._attached); - } - parent() { - return this._parentLink?.container; - } - isDeleted() { - return this._doc?._isContainerDeleted(this) ?? false; - } - subscribe(listener) { - return this._doc?._subscribeContainer(this, listener) ?? (() => { }); - } - getShallowValue() { - return this.toJSON(); - } - _attach(doc, id, parent) { - this._doc = doc; - this._codecId = id; - this._containerId = formatContainerId(id); - this._parentLink = parent === undefined ? undefined : { container: parent }; - } - _setParentBinding(parent, binding) { - this._parentLink = { container: parent, binding }; - } -} -export class LoroMap extends LoroContainer { - _entries = new Map(); - _keyIndex = new OrderedIndex((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0); - _keyRecords = new Map(); - kind() { - return "Map"; - } - get(key) { - const record = this._entries.get(key); - return record === undefined || record.deleted - ? undefined - : cloneRuntimeValue(record.value); - } - set(key, value) { - if (isContainer(value)) { - throw new TypeError("use setContainer() to attach a child container"); - } - if (this._doc === undefined) { - this._applyValue(key, normalizeDetachedValue(value), { peer: 0n, lamport: 0 }); - return; - } - this._doc._mapSet(this, key, value); - } - insert(key, value) { - this.set(key, value); - } - delete(key) { - if (this._doc === undefined) { - if (this._entries.has(key)) - this._removeVisibleKey(key); - this._entries.delete(key); - return; - } - this._doc._mapDelete(this, key); - } - clear() { - for (const key of this.keys()) - this.delete(key); - } - keys() { - return this._keyIndex.values().map(({ key }) => key); - } - values() { - const records = this._keyIndex.values(); - const output = new Array(records.length); - for (let index = 0; index < records.length; index += 1) { - output[index] = this.get(records[index].key); - } - return output; - } - entries() { - const records = this._keyIndex.values(); - const output = new Array(records.length); - for (let index = 0; index < records.length; index += 1) { - output[index] = [records[index].key, this.get(records[index].key)]; - } - return output; - } - get size() { - return this._keyIndex.size; - } - setContainer(key, child) { - if (this._doc === undefined) { - this._applyValue(key, child, { peer: 0n, lamport: 0 }); - return child; - } - return this._doc._mapSetContainer(this, key, child); - } - insertContainer(key, child) { - return this.setContainer(key, child); - } - getOrCreateContainer(key, child) { - const current = this.get(key); - if (current === undefined || current === null) - return this.setContainer(key, child); - if (isContainer(current) && current.kind() === child.kind()) - return current; - throw new TypeError(`map key ${key} already contains an incompatible value`); - } - ensureMergeableMap(key) { - return this._ensureMergeable(key, "Map"); - } - ensureMergeableList(key) { - return this._ensureMergeable(key, "List"); - } - ensureMergeableMovableList(key) { - return this._ensureMergeable(key, "MovableList"); - } - ensureMergeableText(key) { - return this._ensureMergeable(key, "Text"); - } - ensureMergeableTree(key) { - return this._ensureMergeable(key, "Tree"); - } - ensureMergeableCounter(key) { - return this._ensureMergeable(key, "Counter"); - } - getLastEditor(key) { - return this._entries.get(key)?.writer.peer.toString(); - } - toJSON() { - const output = {}; - for (const { key } of this._keyIndex.values()) { - output[key] = runtimeValueToJson(this._entries.get(key).value); - } - return output; - } - getShallowValue() { - const output = {}; - for (const { key } of this._keyIndex.values()) { - output[key] = runtimeValueToShallow(this._entries.get(key).value); - } - return output; - } - _applyValue(key, value, writer, rawValue = value) { - const current = this._entries.get(key); - if (current !== undefined && compareWriters(current.writer, writer) > 0) - return; - if (current === undefined || current.deleted) - this._addVisibleKey(key); - this._entries.set(key, { value, rawValue, deleted: false, writer }); - if (value instanceof LoroContainer) { - value._setParentBinding(this, { kind: "map", key }); - } - } - _applyDelete(key, writer) { - const current = this._entries.get(key); - if (current !== undefined && compareWriters(current.writer, writer) > 0) - return; - if (current !== undefined && !current.deleted) - this._removeVisibleKey(key); - this._entries.set(key, { - value: undefined, - rawValue: undefined, - deleted: true, - writer, - }); - } - _replaceRecord(key, record) { - const current = this._entries.get(key); - if (current !== undefined && !current.deleted) - this._removeVisibleKey(key); - if (record === undefined) { - this._entries.delete(key); - return; - } - this._entries.set(key, record); - if (!record.deleted) { - this._addVisibleKey(key); - if (record.value instanceof LoroContainer) { - record.value._setParentBinding(this, { kind: "map", key }); - } - } - } - _reset() { - this._entries.clear(); - this._keyIndex.clear(); - this._keyRecords.clear(); - } - _addVisibleKey(key) { - let record = this._keyRecords.get(key); - if (record === undefined) { - record = { key }; - this._keyRecords.set(key, record); - } - this._keyIndex.add(record); - } - _removeVisibleKey(key) { - const record = this._keyRecords.get(key); - if (record !== undefined) - this._keyIndex.delete(record); - } - _ensureMergeable(key, type) { - if (this._doc === undefined) { - throw new Error("cannot ensure a mergeable child on a detached map"); - } - return this._doc._mapEnsureMergeable(this, key, type); - } -} -export class LoroList extends LoroContainer { - _sequence = new SequenceIndex(); - _detachedCounter = 0; - get _elements() { - return this._sequence.all(); - } - kind() { - return "List"; - } - get length() { - return this._sequence.visibleLength; - } - get(index) { - return cloneRuntimeValue(this._sequence.atVisible(index)?.value); - } - toArray() { - return this._visibleElements().map((element) => cloneRuntimeValue(element.value)); - } - insert(pos, value) { - this._validateInsertPosition(pos); - if (isContainer(value)) - throw new TypeError("use insertContainer() to attach a child container"); - if (this._doc === undefined) { - this._insertVisible(pos, [normalizeDetachedValue(value)], [this._nextDetachedId()], [0]); - return; - } - this._doc._listInsert(this, pos, value); - } - push(value) { - this.insert(this.length, value); - } - delete(pos, len) { - validateRange(pos, len, this.length); - if (len === 0) - return; - if (this._doc === undefined) { - for (const element of this._sequence.visibleRange(pos, pos + len)) - this._sequence.setDeleted(element, true); - return; - } - this._doc._sequenceDelete(this, pos, len); - } - clear() { - this.delete(0, this.length); - } - pop() { - if (this.length === 0) - return undefined; - const value = this.get(this.length - 1); - this.delete(this.length - 1, 1); - return value; - } - insertContainer(pos, child) { - this._validateInsertPosition(pos); - if (this._doc === undefined) { - this._insertVisible(pos, [child], [this._nextDetachedId()], [0]); - return child; - } - return this._doc._listInsertContainer(this, pos, child); - } - pushContainer(child) { - return this.insertContainer(this.length, child); - } - getIdAt(pos) { - const id = this._sequence.atVisible(pos)?.id; - return id === undefined - ? undefined - : { peer: id.peer.toString(), counter: id.counter }; - } - getCursor(pos, side = 0) { - if (!Number.isSafeInteger(pos) || pos < 0) - return undefined; - if (pos >= this.length) { - return new Cursor(this.id, this._sequence.atVisible(this.length - 1)?.id, 1, this.length); - } - return new Cursor(this.id, this._sequence.atVisible(pos).id, side, pos); - } - toJSON() { - return this._visibleElements().map((element) => runtimeValueToJson(element.value)); - } - getShallowValue() { - return this._visibleElements().map((element) => runtimeValueToShallow(element.value)); - } - _visibleElements() { - return this._sequence.visible(); - } - _visibleElementsRange(start, end) { - return this._sequence.visibleRange(start, end); - } - _valuesRange(start, end) { - return this._sequence - .visibleRange(start, end) - .map((element) => cloneRuntimeValue(element.value)); - } - _visibleElementAt(position) { - return this._sequence.atVisible(position); - } - _insertVisible(position, values, ids, lamports) { - const movable = this instanceof LoroMovableList; - const elements = values.map((value, index) => { - const id = ids[index]; - const lamport = lamports[index]; - return { - value, - id, - lamport, - deleted: false, - originLeft: undefined, - originRight: undefined, - valueHistory: movable ? [{ id, lamport, value }] : undefined, - }; - }); - this._sequence.insertAtVisible(position, elements); - this._bindChildren(elements); - } - _insertFugue(position, values, ids, lamports, causalVersion) { - const movable = this instanceof LoroMovableList; - const elements = values.map((value, index) => { - const id = ids[index]; - const elementLamport = lamports[index]; - return { - value, - id, - lamport: elementLamport, - deleted: false, - originLeft: undefined, - originRight: undefined, - valueHistory: movable ? [{ id, lamport: elementLamport, value }] : undefined, - }; - }); - insertFugueElements(this._sequence, position, elements, causalVersion, - // Moves break the origin-tree preorder used by the direct-child index. - !(this instanceof LoroMovableList)); - this._bindChildren(elements); - } - _deleteIdSpan(startId, length, deletedBy) { - if (Math.abs(length) === 1) { - const element = this._sequence.findById(startId); - if (element === undefined) - return; - this._sequence.deleteElement(element, deletedBy); - return; - } - this._sequence.deleteIdSpan(startId, length, deletedBy); - } - _validateInsertPosition(position) { - if (!Number.isSafeInteger(position) || position < 0 || position > this.length) { - throw new RangeError(`list position ${position} is out of range`); - } - } - _nextDetachedId() { - return { peer: 0n, counter: this._detachedCounter++ }; - } - _reset() { - this._sequence.reset(); - this._detachedCounter = 0; - } - _bindChildren(elements) { - for (const element of elements) { - if (element.value instanceof LoroContainer) { - element.value._setParentBinding(this, { kind: "sequence", element }); - } - } - } -} -const TEXT_NUMBER_STRIDE = 6; -const TEXT_PEER_STRIDE = 4; -const TEXT_COUNTER = 0; -const TEXT_LAMPORT = 1; -const TEXT_UTF16_END = 2; -const TEXT_ORIGIN_LEFT_COUNTER = 3; -const TEXT_ORIGIN_RIGHT_COUNTER = 4; -const TEXT_DELETED_BY_COUNTER = 5; -const TEXT_ID_PEER = 0; -const TEXT_ORIGIN_LEFT_PEER = 1; -const TEXT_ORIGIN_RIGHT_PEER = 2; -const TEXT_DELETED_BY_PEER = 3; -const NO_DELETION_IDS = []; -class TextSequenceSpan extends SequenceSpan { - #text = ""; - #length = 0; - #numbers; - #peers; - #pendingStartPeer; - #pendingStartCounter = 0; - #pendingLamport = 0; - #pendingOriginLeftPeer; - #pendingOriginLeftCounter = 0; - #pendingOriginRightPeer; - #pendingOriginRightCounter = 0; - #deletedBits = 0; - #retained; - #deletedBy; - #attributes; - #attributeMeta; - #attributeHistory; - #valueHistory; - #moveHistory; - static fromText(text, startId, lamport, originLeft, originRight) { - return TextSequenceSpan.fromTextOrigins(text, startId, lamport, originLeft?.peer, originLeft?.counter ?? 0, originRight?.peer, originRight?.counter ?? 0); - } - static fromTextOrigins(text, startId, lamport, originLeftPeer, originLeftCounter, originRightPeer, originRightCounter) { - const span = new TextSequenceSpan(); - span.#text = text; - for (const _value of text) - span.#length += 1; - span.#pendingStartPeer = startId.peer; - span.#pendingStartCounter = startId.counter; - span.#pendingLamport = lamport; - span.#pendingOriginLeftPeer = originLeftPeer; - span.#pendingOriginLeftCounter = originLeftCounter; - span.#pendingOriginRightPeer = originRightPeer; - span.#pendingOriginRightCounter = originRightCounter; - return span; - } - /** The caller must have validated ID uniqueness across the complete snapshot. */ - static fromValidatedSnapshotChunk(text, peers, counters, lamports) { - if (peers.length === 0 || - peers.length !== counters.length || - peers.length !== lamports.length) { - throw new Error("invalid text snapshot chunk columns"); - } - const span = new TextSequenceSpan(); - span.#text = text; - span.#length = peers.length; - const numbers = new Array(peers.length * TEXT_NUMBER_STRIDE).fill(0); - const peerColumns = new Array(peers.length * TEXT_PEER_STRIDE).fill(undefined); - let utf16End = 0; - let offset = 0; - for (const value of text) { - if (offset >= peers.length) { - throw new Error("text snapshot chunk contains too many Unicode scalars"); - } - utf16End += value.length; - const numberBase = offset * TEXT_NUMBER_STRIDE; - numbers[numberBase + TEXT_COUNTER] = counters[offset]; - numbers[numberBase + TEXT_LAMPORT] = lamports[offset]; - numbers[numberBase + TEXT_UTF16_END] = utf16End; - const peerBase = offset * TEXT_PEER_STRIDE; - peerColumns[peerBase + TEXT_ID_PEER] = peers[offset]; - offset += 1; - } - if (offset !== peers.length) { - throw new Error("text snapshot chunk contains too few Unicode scalars"); - } - span.#numbers = numbers; - span.#peers = peerColumns; - return span; - } - get length() { - return this.#length; - } - idsAreUnique() { - return true; - } - elementAt(offset) { - return this.#retained?.get(offset) ?? new PackedTextElement(this, offset); - } - idAt(offset) { - return { - peer: this.peerAt(offset), - counter: this.counterAt(offset), - }; - } - peerAt(offset) { - return this.#peer(offset, TEXT_ID_PEER); - } - counterAt(offset) { - return this.#number(offset, TEXT_COUNTER); - } - lamportAt(offset) { - return this.#number(offset, TEXT_LAMPORT); - } - deletedAt(offset) { - return ((this.#deletedBits >>> offset) & 1) !== 0; - } - setDeletedAt(offset, deleted) { - const bit = 1 << offset; - this.#deletedBits = deleted - ? (this.#deletedBits | bit) >>> 0 - : (this.#deletedBits & ~bit) >>> 0; - } - metricsAt(offset) { - const value = this.valueAt(offset); - return { utf16: value.length, utf8: utf8CodePointLength(value) }; - } - slice(start, end) { - this.#materializeColumns(); - const output = new TextSequenceSpan(); - output.#numbers = []; - output.#peers = []; - output.#length = end - start; - const utf16Start = start === 0 ? 0 : this.#number(start - 1, TEXT_UTF16_END); - const utf16End = end === 0 ? 0 : this.#number(end - 1, TEXT_UTF16_END); - output.#text = this.#text.slice(utf16Start, utf16End); - for (let offset = start; offset < end; offset += 1) { - const numberBase = offset * TEXT_NUMBER_STRIDE; - for (let column = 0; column < TEXT_NUMBER_STRIDE; column += 1) { - const value = this.#numbers[numberBase + column]; - output.#numbers.push(column === TEXT_UTF16_END ? value - utf16Start : value); - } - const peerBase = offset * TEXT_PEER_STRIDE; - output.#peers.push(this.#peers[peerBase], this.#peers[peerBase + 1], this.#peers[peerBase + 2], this.#peers[peerBase + 3]); - const retained = this.#retained?.get(offset); - if (retained !== undefined) { - const nextOffset = offset - start; - retained._retarget(output, nextOffset); - (output.#retained ??= new Map()).set(nextOffset, retained); - } - } - const length = end - start; - const mask = length === 32 ? 4294967295 : (1 << length) - 1; - output.#deletedBits = (this.#deletedBits >>> start) & mask; - output.#deletedBy = copyOffsetMap(this.#deletedBy, start, end); - output.#attributes = copyOffsetMap(this.#attributes, start, end); - output.#attributeMeta = copyOffsetMap(this.#attributeMeta, start, end); - output.#attributeHistory = copyOffsetMap(this.#attributeHistory, start, end); - output.#valueHistory = copyOffsetMap(this.#valueHistory, start, end); - output.#moveHistory = copyOffsetMap(this.#moveHistory, start, end); - return output; - } - append(other) { - if (!(other instanceof TextSequenceSpan)) - return false; - const oldLength = this.length; - const utf16Base = this.#text.length; - this.#materializeColumns(); - this.#text += other.#text; - for (let offset = 0; offset < other.length; offset += 1) { - for (let column = 0; column < TEXT_NUMBER_STRIDE; column += 1) { - const value = other.#number(offset, column); - this.#numbers.push(column === TEXT_UTF16_END ? value + utf16Base : value); - } - this.#peers.push(other.#peer(offset, 0), other.#peer(offset, 1), other.#peer(offset, 2), other.#peer(offset, 3)); - const retained = other.#retained?.get(offset); - if (retained !== undefined) { - const nextOffset = oldLength + offset; - retained._retarget(this, nextOffset); - (this.#retained ??= new Map()).set(nextOffset, retained); - } - } - this.#deletedBits = (this.#deletedBits | (other.#deletedBits << oldLength)) >>> 0; - this.#length += other.length; - this.#deletedBy = appendOffsetMap(this.#deletedBy, other.#deletedBy, oldLength); - this.#attributes = appendOffsetMap(this.#attributes, other.#attributes, oldLength); - this.#attributeMeta = appendOffsetMap(this.#attributeMeta, other.#attributeMeta, oldLength); - this.#attributeHistory = appendOffsetMap(this.#attributeHistory, other.#attributeHistory, oldLength); - this.#valueHistory = appendOffsetMap(this.#valueHistory, other.#valueHistory, oldLength); - this.#moveHistory = appendOffsetMap(this.#moveHistory, other.#moveHistory, oldLength); - return true; - } - deletionIdsAt(offset) { - const deletedBy = this.#deletedBy?.get(offset); - const peer = this.#peer(offset, TEXT_DELETED_BY_PEER); - if (peer === undefined) - return deletedBy ?? NO_DELETION_IDS; - const length = deletedBy?.length ?? 0; - const output = new Array(length + 1); - output[0] = { - peer, - counter: this.#number(offset, TEXT_DELETED_BY_COUNTER), - }; - for (let index = 0; index < length; index += 1) { - output[index + 1] = deletedBy[index]; - } - return output; - } - retain(offset, element) { - if (element instanceof PackedTextElement) { - (this.#retained ??= new Map()).set(offset, element); - } - } - forEachRetained(visit) { - for (const [offset, element] of this.#retained ?? []) - visit(element, offset); - } - valueAt(offset) { - const start = offset === 0 ? 0 : this.#number(offset - 1, TEXT_UTF16_END); - return this.#text.slice(start, this.#number(offset, TEXT_UTF16_END)); - } - textRange(start, end) { - const utf16Start = start === 0 ? 0 : this.#number(start - 1, TEXT_UTF16_END); - const utf16End = end === 0 ? 0 : this.#number(end - 1, TEXT_UTF16_END); - return this.#text.slice(utf16Start, utf16End); - } - setValueAt(offset, value) { - const current = this.valueAt(offset); - if (current === value) - return; - this.#materializeColumns(); - const start = offset === 0 ? 0 : this.#number(offset - 1, TEXT_UTF16_END); - const end = this.#number(offset, TEXT_UTF16_END); - this.#text = this.#text.slice(0, start) + value + this.#text.slice(end); - const difference = value.length - (end - start); - for (let index = offset; index < this.length; index += 1) { - this.#setNumber(index, TEXT_UTF16_END, this.#number(index, TEXT_UTF16_END) + difference); - } - } - originLeftAt(offset) { - return this.#idColumnAt(offset, TEXT_ORIGIN_LEFT_PEER, TEXT_ORIGIN_LEFT_COUNTER); - } - setOriginLeftAt(offset, id) { - this.#setIdColumn(offset, TEXT_ORIGIN_LEFT_PEER, TEXT_ORIGIN_LEFT_COUNTER, id); - } - originRightAt(offset) { - return this.#idColumnAt(offset, TEXT_ORIGIN_RIGHT_PEER, TEXT_ORIGIN_RIGHT_COUNTER); - } - setOriginRightAt(offset, id) { - this.#setIdColumn(offset, TEXT_ORIGIN_RIGHT_PEER, TEXT_ORIGIN_RIGHT_COUNTER, id); - } - deletedByPeerAt(offset) { - return this.#peer(offset, TEXT_DELETED_BY_PEER); - } - setDeletedByPeerAt(offset, peer) { - this.#setPeer(offset, TEXT_DELETED_BY_PEER, peer); - } - deletedByCounterAt(offset) { - return this.#peer(offset, TEXT_DELETED_BY_PEER) === undefined - ? undefined - : this.#number(offset, TEXT_DELETED_BY_COUNTER); - } - setDeletedByCounterAt(offset, counter) { - this.#setNumber(offset, TEXT_DELETED_BY_COUNTER, counter ?? 0); - } - deletedByAt(offset) { - return this.#deletedBy?.get(offset); - } - setDeletedByAt(offset, ids) { - this.#deletedBy = setOffsetMap(this.#deletedBy, offset, ids); - } - attributesAt(offset) { - return this.#attributes?.get(offset); - } - setAttributesAt(offset, value) { - this.#attributes = setOffsetMap(this.#attributes, offset, value); - } - attributeMetaAt(offset) { - return this.#attributeMeta?.get(offset); - } - setAttributeMetaAt(offset, value) { - this.#attributeMeta = setOffsetMap(this.#attributeMeta, offset, value); - } - attributeHistoryAt(offset) { - return this.#attributeHistory?.get(offset); - } - setAttributeHistoryAt(offset, value) { - this.#attributeHistory = setOffsetMap(this.#attributeHistory, offset, value); - } - valueHistoryAt(offset) { - return this.#valueHistory?.get(offset); - } - setValueHistoryAt(offset, value) { - this.#valueHistory = setOffsetMap(this.#valueHistory, offset, value); - } - moveHistoryAt(offset) { - return this.#moveHistory?.get(offset); - } - setMoveHistoryAt(offset, value) { - this.#moveHistory = setOffsetMap(this.#moveHistory, offset, value); - } - #number(offset, column) { - if (this.#numbers !== undefined) { - return this.#numbers[offset * TEXT_NUMBER_STRIDE + column]; - } - switch (column) { - case TEXT_COUNTER: - return this.#pendingStartCounter + offset; - case TEXT_LAMPORT: - return this.#pendingLamport + offset; - case TEXT_UTF16_END: { - let utf16End = 0; - let index = 0; - for (const value of this.#text) { - utf16End += value.length; - if (index === offset) - return utf16End; - index += 1; - } - throw new RangeError("text span offset is out of range"); - } - case TEXT_ORIGIN_LEFT_COUNTER: - return offset === 0 - ? this.#pendingOriginLeftCounter - : this.#pendingStartCounter + offset - 1; - case TEXT_ORIGIN_RIGHT_COUNTER: - return this.#pendingOriginRightCounter; - case TEXT_DELETED_BY_COUNTER: - return 0; - default: - throw new RangeError("text span number column is out of range"); - } - } - #setNumber(offset, column, value) { - this.#materializeColumns(); - this.#numbers[offset * TEXT_NUMBER_STRIDE + column] = value; - } - #peer(offset, column) { - if (this.#peers !== undefined) { - return this.#peers[offset * TEXT_PEER_STRIDE + column]; - } - switch (column) { - case TEXT_ID_PEER: - return this.#pendingStartPeer; - case TEXT_ORIGIN_LEFT_PEER: - return offset === 0 ? this.#pendingOriginLeftPeer : this.#pendingStartPeer; - case TEXT_ORIGIN_RIGHT_PEER: - return this.#pendingOriginRightPeer; - case TEXT_DELETED_BY_PEER: - return undefined; - default: - throw new RangeError("text span peer column is out of range"); - } - } - #setPeer(offset, column, value) { - this.#materializeColumns(); - this.#peers[offset * TEXT_PEER_STRIDE + column] = value; - } - #materializeColumns() { - if (this.#numbers !== undefined) - return; - const numbers = []; - const peers = []; - let utf16End = 0; - let offset = 0; - for (const value of this.#text) { - utf16End += value.length; - numbers.push(this.#pendingStartCounter + offset, this.#pendingLamport + offset, utf16End, offset === 0 - ? this.#pendingOriginLeftCounter - : this.#pendingStartCounter + offset - 1, this.#pendingOriginRightCounter, 0); - peers.push(this.#pendingStartPeer, offset === 0 ? this.#pendingOriginLeftPeer : this.#pendingStartPeer, this.#pendingOriginRightPeer, undefined); - offset += 1; - } - this.#numbers = numbers; - this.#peers = peers; - } - #idColumnAt(offset, peerColumn, counterColumn) { - const peer = this.#peer(offset, peerColumn); - return peer === undefined - ? undefined - : { peer, counter: this.#number(offset, counterColumn) }; - } - #setIdColumn(offset, peerColumn, counterColumn, id) { - this.#setPeer(offset, peerColumn, id?.peer); - this.#setNumber(offset, counterColumn, id?.counter ?? 0); - } -} -class PackedTextElement { - #span; - #offset; - peer; - counter; - constructor(span, offset) { - this.#span = span; - this.#offset = offset; - const id = span.idAt(offset); - this.peer = id.peer; - this.counter = id.counter; - } - get id() { - return this; - } - get lamport() { - return this.#span.lamportAt(this.#offset); - } - set lamport(_value) { - throw new TypeError("text element lamport is immutable"); - } - get value() { - return this.#span.valueAt(this.#offset); - } - set value(value) { - this.#span.setValueAt(this.#offset, value); - } - get deleted() { - return this.#span.deletedAt(this.#offset); - } - set deleted(value) { - this.#span.setDeletedAt(this.#offset, value); - } - get originLeft() { - return this.#span.originLeftAt(this.#offset); - } - set originLeft(value) { - this.#span.setOriginLeftAt(this.#offset, value); - } - get originRight() { - return this.#span.originRightAt(this.#offset); - } - set originRight(value) { - this.#span.setOriginRightAt(this.#offset, value); - } - get deletedByPeer() { - return this.#span.deletedByPeerAt(this.#offset); - } - set deletedByPeer(value) { - this.#span.setDeletedByPeerAt(this.#offset, value); - } - get deletedByCounter() { - return this.#span.deletedByCounterAt(this.#offset); - } - set deletedByCounter(value) { - this.#span.setDeletedByCounterAt(this.#offset, value); - } - get deletedBy() { - return this.#span.deletedByAt(this.#offset); - } - set deletedBy(value) { - this.#span.setDeletedByAt(this.#offset, value); - } - get attributes() { - return this.#span.attributesAt(this.#offset); - } - set attributes(value) { - this.#span.setAttributesAt(this.#offset, value); - } - get attributeMeta() { - return this.#span.attributeMetaAt(this.#offset); - } - set attributeMeta(value) { - this.#span.setAttributeMetaAt(this.#offset, value); - } - get attributeHistory() { - return this.#span.attributeHistoryAt(this.#offset); - } - set attributeHistory(value) { - this.#span.setAttributeHistoryAt(this.#offset, value); - } - get valueHistory() { - return this.#span.valueHistoryAt(this.#offset); - } - set valueHistory(value) { - this.#span.setValueHistoryAt(this.#offset, value); - } - get moveHistory() { - return this.#span.moveHistoryAt(this.#offset); - } - set moveHistory(value) { - this.#span.setMoveHistoryAt(this.#offset, value); - } - _retarget(span, offset) { - this.#span = span; - this.#offset = offset; - } -} -function setOffsetMap(map, offset, value) { - if (value === undefined) { - map?.delete(offset); - return map; - } - const output = map ?? new Map(); - output.set(offset, value); - return output; -} -function copyOffsetMap(map, start, end) { - if (map === undefined) - return undefined; - const output = new Map(); - for (const [offset, value] of map) { - if (offset >= start && offset < end) - output.set(offset - start, value); - } - return output.size === 0 ? undefined : output; -} -function appendOffsetMap(left, right, offset) { - if (right === undefined) - return left; - const output = left ?? new Map(); - for (const [index, value] of right) - output.set(offset + index, value); - return output; -} -/** Reads a visible range from sequence storage without materializing scalar views. */ -function storageTextRange(storage, start, end) { - if (storage instanceof TextSequenceSpan) - return storage.textRange(start, end); - if (storage instanceof SequenceSpan) { - throw new Error("unexpected text sequence span implementation"); - } - if (Array.isArray(storage)) { - const elements = storage; - let text = ""; - for (let offset = start; offset < end; offset += 1) - text += elements[offset].value; - return text; - } - return storage.value; -} -export class LoroText extends LoroContainer { - _sequence = new SequenceIndex((element) => ({ - utf16: element.value.length, - utf8: utf8CodePointLength(element.value), - })); - _detachedCounter = 0; - _detachedStyleCounter = 0; - _attributeHistoryComplete = true; - _styleIndex = new TextStyleIndex(); - _styleVersion; - #insertionIdContext = { - leftPeer: undefined, - leftCounter: 0, - startIndex: 0, - rightPeer: undefined, - rightCounter: 0, - }; - #validatedSnapshotSpans; - #attributeValuesCache = new WeakMap(); - get _elements() { - return this._sequence.all(); - } - kind() { - return "Text"; - } - get length() { - return this._sequence.visibleUtf16Length; - } - toString() { - return this._stringRange(0, this._sequence.visibleLength); - } - _stringRange(start, end) { - const chunks = []; - let cursor = 0; - this._sequence.forEachVisibleStorageRange((storage, rangeStart, rangeEnd) => { - const nextCursor = cursor + (rangeEnd - rangeStart); - if (nextCursor > start && end > cursor) { - chunks.push(storageTextRange(storage, start > cursor ? rangeStart + (start - cursor) : rangeStart, end < nextCursor ? rangeStart + (end - cursor) : rangeEnd)); - } - cursor = nextCursor; - return nextCursor >= end ? false : undefined; - }); - return chunks.join(""); - } - iter(callback) { - this._sequence.forEachVisibleStorageRange((storage, start, end) => callback(storageTextRange(storage, start, end)) === false ? false : undefined); - } - insert(pos, text) { - const unicodePosition = this._validateInsertPosition(pos); - if (text.length === 0) - return; - if (this._doc === undefined) { - const span = TextSequenceSpan.fromText(text, { peer: 0n, counter: this._detachedCounter }, 0, undefined, undefined); - this._detachedCounter += span.length; - this._sequence.insertSpanAtVisible(unicodePosition, span); - return; - } - this._doc._textInsert(this, unicodePosition, text); - } - push(text) { - this.insert(this.length, text); - } - insertUtf8(index, text) { - const utf16 = this.convertPos(index, "utf8", "utf16"); - if (utf16 === undefined) - throw new RangeError("UTF-8 position is out of range"); - this.insert(utf16, text); - } - delete(pos, len) { - validateRange(pos, len, this.length); - if (len === 0) - return; - const start = this._unicodePosition(pos); - const end = this._unicodePosition(pos + len); - if (this._doc === undefined) { - for (const element of this._sequence.visibleRange(start, end)) - this._sequence.setDeleted(element, true); - return; - } - this._doc._textDelete(this, start, end - start); - } - deleteUtf8(index, length) { - const start = this.convertPos(index, "utf8", "utf16"); - const end = this.convertPos(index + length, "utf8", "utf16"); - if (start === undefined || end === undefined) - throw new RangeError("UTF-8 range is out of bounds"); - this.delete(start, end - start); - } - slice(start, end) { - validateRange(start, end - start, this.length); - const unicodeStart = this._unicodePosition(start); - const unicodeEnd = this._unicodePosition(end); - return this._stringRange(unicodeStart, unicodeEnd); - } - charAt(pos) { - if (pos === this.length) - return ""; - return this._sequence.atVisible(this._unicodePosition(pos))?.value ?? ""; - } - splice(pos, len, text) { - const removed = this.slice(pos, pos + len); - this.delete(pos, len); - this.insert(pos, text); - return removed; - } - mark(range, key, value) { - validateRange(range.start, range.end - range.start, this.length); - const start = this._unicodePosition(range.start); - const end = this._unicodePosition(range.end); - if (this._doc === undefined) { - this._applyMark(start, end, key, normalizeDetachedValue(value)); - return; - } - this._doc._textMark(this, start, end, key, value); - } - unmark(range, key) { - validateRange(range.start, range.end - range.start, this.length); - const start = this._unicodePosition(range.start); - const end = this._unicodePosition(range.end); - if (!this._styleIndex.rangeHasKey(this._styleRuns(start, end), key, this._styleVersion)) { - return; - } - this.mark(range, key, null); - } - applyDelta(delta) { - let position = 0; - const marks = []; - for (const operation of delta) { - if ("insert" in operation) { - this.insert(position, operation.insert); - const length = operation.insert.length; - if (operation.attributes !== undefined) { - marks.push({ - start: position, - end: position + length, - attributes: operation.attributes, - }); - } - position += length; - } - else if ("delete" in operation) { - this.delete(position, operation.delete); - } - else { - if (operation.attributes !== undefined) { - marks.push({ - start: position, - end: position + operation.retain, - attributes: operation.attributes, - }); - } - position += operation.retain; - } - } - for (const { start, end, attributes } of marks) { - for (const [key, value] of Object.entries(attributes)) { - this.mark({ start, end }, key, value); - } - } - } - toDelta() { - return textElementsToDelta(this._visibleElements(), this.#attributeResolver()); - } - sliceDelta(start, end) { - validateRange(start, end - start, this.length); - const unicodeStart = this._unicodePosition(start); - const unicodeEnd = this._unicodePosition(end); - return textElementsToDelta(this._sequence.visibleRange(unicodeStart, unicodeEnd), this.#attributeResolver()); - } - sliceDeltaUtf8(start, end) { - const utf16Start = this.convertPos(start, "utf8", "utf16"); - const utf16End = this.convertPos(end, "utf8", "utf16"); - if (utf16Start === undefined || utf16End === undefined) { - throw new RangeError("UTF-8 range is out of bounds"); - } - return this.sliceDelta(utf16Start, utf16End); - } - convertPos(index, from, to) { - if (!isTextPosType(from) || !isTextPosType(to)) - return undefined; - const visibleLength = this._sequence.visibleLength; - const directMetric = from === "unicode" || - (from === "utf16" && this._sequence.visibleUtf16Length === visibleLength) || - (from === "utf8" && this._sequence.visibleUtf8Length === visibleLength); - const unicodeIndex = directMetric - ? Number.isSafeInteger(index) && index >= 0 && index <= visibleLength - ? index - : undefined - : this._sequence.visibleIndexAtMetricOffset(index, from); - if (unicodeIndex === undefined) - return undefined; - if (to === "unicode" || - (to === "utf16" && this._sequence.visibleUtf16Length === visibleLength) || - (to === "utf8" && this._sequence.visibleUtf8Length === visibleLength)) { - return unicodeIndex; - } - return this._sequence.metricOffsetAtVisibleIndex(unicodeIndex, to); - } - getCursor(pos, side = 0) { - if (!Number.isSafeInteger(pos) || pos < 0) - return undefined; - if (pos >= this.length) { - return new Cursor(this.id, this._sequence.atVisible(this._sequence.visibleLength - 1)?.id, 1, this.length); - } - const unicodePosition = this.convertPos(pos, "utf16", "unicode"); - if (unicodePosition === undefined) - return undefined; - return new Cursor(this.id, this._sequence.atVisible(unicodePosition).id, side, pos); - } - getEditorOf(pos) { - const unicodePosition = this.convertPos(pos, "utf16", "unicode"); - return unicodePosition === undefined - ? undefined - : this._sequence.atVisible(unicodePosition)?.id.peer.toString(); - } - update(text, _options) { - updateText(this, text); - } - updateByLine(text, options) { - this.update(text, options); - } - toJSON() { - return this.toString(); - } - getShallowValue() { - return this.toString(); - } - _visibleElements() { - return this._sequence.visible(); - } - _visibleElementsRange(start, end) { - return this._sequence.visibleRange(start, end); - } - _visibleElementAt(position) { - return this._sequence.atVisible(position); - } - _insertVisible(position, characters, ids, lamports) { - this._sequence.insertAtVisible(position, characters.map((value, index) => ({ - value, - id: ids[index], - lamport: lamports[index], - deleted: false, - originLeft: undefined, - originRight: undefined, - }))); - } - _insertValidatedSnapshotChunk(text, peers, counters, lamports) { - const span = TextSequenceSpan.fromValidatedSnapshotChunk(text, peers, counters, lamports); - if (this.#validatedSnapshotSpans !== undefined) { - this.#validatedSnapshotSpans.push(span); - } - else { - this._sequence.insertSpanAtPhysical(this._sequence.allLength, span); - } - } - _beginValidatedSnapshotLoad() { - if (this.#validatedSnapshotSpans !== undefined) { - throw new Error("text snapshot load is already active"); - } - this.#validatedSnapshotSpans = []; - } - _endValidatedSnapshotLoad() { - const spans = this.#validatedSnapshotSpans; - if (spans === undefined) - throw new Error("text snapshot load is not active"); - this.#validatedSnapshotSpans = undefined; - this._sequence.loadValidatedSpans(spans); - } - _forEachVisibleSnapshotData(appendText, appendId) { - this._sequence.forEachVisibleStorageRange((storage, start, end) => { - if (storage instanceof TextSequenceSpan) { - appendText(storage.textRange(start, end), end - start); - for (let offset = start; offset < end; offset += 1) { - appendId(storage.peerAt(offset), storage.counterAt(offset), storage.lamportAt(offset)); - } - return; - } - if (storage instanceof SequenceSpan) { - throw new Error("unexpected text sequence span implementation"); - } - const elements = Array.isArray(storage) ? storage : [storage]; - let text = ""; - for (let offset = start; offset < end; offset += 1) { - const element = elements[offset]; - text += element.value; - appendId(element.id.peer, element.id.counter, element.lamport); - } - appendText(text, end - start); - }); - } - _insertFugue(position, text, startId, lamport, causalVersion) { - const current = this._sequence.isFullyIncluded(causalVersion); - const authored = current ? undefined : this._sequence.causalView(causalVersion); - const authoredLength = current ? this._sequence.visibleLength : authored.length; - const authoredPosition = Math.min(position, authoredLength); - const needsStyleElements = !this._styleIndex.isEmpty; - const firstCodePoint = text.codePointAt(0); - const singleScalar = text.length === 1 || (text.length === 2 && firstCodePoint > 0xffff); - const useIdContext = current && !needsStyleElements && !singleScalar; - const currentContext = current && !useIdContext - ? this._sequence.visibleInsertionContext(authoredPosition) - : undefined; - const currentIdContext = useIdContext - ? this._sequence.visibleInsertionIdContext(authoredPosition, this.#insertionIdContext) - : undefined; - const authoredLeft = current - ? currentContext?.left - : authored.at(authoredPosition - 1); - const authoredRight = needsStyleElements - ? current - ? this._sequence.atVisible(authoredPosition) - : authored.at(authoredPosition) - : undefined; - const insertion = currentIdContext === undefined - ? fugueInsertion(this._sequence, position, startId, causalVersion, true, currentContext ?? { current: false, left: authoredLeft }) - : undefined; - const insertIndex = currentIdContext?.startIndex ?? insertion.insertIndex; - let inherited; - if (needsStyleElements) { - inherited = new Map(); - for (const [key, meta] of this._attributeMetasAt(authoredLeft, causalVersion)) { - if ((meta.info & 0b100) !== 0) - inherited.set(key, meta); - } - for (const [key, meta] of this._attributeMetasAt(authoredRight, causalVersion)) { - if ((meta.info & 0b010) !== 0) - inherited.set(key, meta); - } - } - let insertedLength; - if (singleScalar) { - const originLeft = currentIdContext === undefined || currentIdContext.leftPeer === undefined - ? insertion?.originLeft - : { - peer: currentIdContext.leftPeer, - counter: currentIdContext.leftCounter, - }; - const originRight = currentIdContext === undefined || currentIdContext.rightPeer === undefined - ? insertion?.originRight - : { - peer: currentIdContext.rightPeer, - counter: currentIdContext.rightCounter, - }; - const element = { - value: text, - id: { peer: startId.peer, counter: startId.counter }, - lamport, - deleted: false, - originLeft, - originRight, - }; - this._sequence.insertAtPhysical(insertIndex, [element]); - if (insertion !== undefined) { - recordFugueInsertion(this._sequence, insertion, [element.id]); - } - insertedLength = 1; - } - else { - const span = currentIdContext === undefined - ? TextSequenceSpan.fromText(text, startId, lamport, insertion.originLeft, insertion.originRight) - : TextSequenceSpan.fromTextOrigins(text, startId, lamport, currentIdContext.leftPeer, currentIdContext.leftCounter, currentIdContext.rightPeer, currentIdContext.rightCounter); - this._sequence.insertSpanAtPhysical(insertIndex, span); - if (insertion !== undefined) { - recordFugueInsertionRun(this._sequence, insertion, startId); - } - insertedLength = span.length; - } - if (inherited !== undefined && inherited.size > 0) { - const insertedRun = [ - { - start: { peer: startId.peer, counter: startId.counter }, - length: insertedLength, - }, - ]; - for (const [key, meta] of inherited) { - this._styleIndex.add(insertedRun, key, meta); - } - } - } - _deleteIdSpan(startId, length, deletedBy) { - if (Math.abs(length) === 1) { - const element = this._sequence.findById(startId); - if (element === undefined) - return; - this._sequence.deleteElement(element, deletedBy); - return; - } - this._sequence.deleteIdSpan(startId, length, deletedBy); - } - _applyMark(start, end, key, value, meta, causalVersion) { - const appliedMeta = meta ?? - ({ - startId: { peer: -1n, counter: this._detachedStyleCounter }, - lamport: this._detachedStyleCounter++, - info: 0, - value, - }); - this._styleIndex.add(this._styleRuns(start, end, causalVersion), key, appliedMeta); - if (this._styleVersion !== undefined) { - const next = new Map(this._styleVersion); - next.set(appliedMeta.startId.peer, Math.max(next.get(appliedMeta.startId.peer) ?? 0, appliedMeta.startId.counter + 1)); - this._styleVersion = next; - } - } - _styleRuns(start, end, causalVersion) { - if (causalVersion === undefined || this._sequence.isFullyIncluded(causalVersion)) { - return this._sequence.visibleIdRuns(start, end); - } - return this._sequence.causalView(causalVersion).idRuns(start, end); - } - _attributeHistoryAt(element, key) { - return this._styleIndex.historyAt(element.id, key); - } - _attributeMetasAt(element, version = this._styleVersion) { - return element === undefined - ? new Map() - : this._styleIndex.metasAt(element.id, version); - } - _attributeMetasResolver(version = this._styleVersion) { - const metasAt = this._styleIndex.resolver(version); - return (element) => metasAt(element.id); - } - _attributesAt(element) { - return this.#attributeValues(this._attributeMetasAt(element)); - } - _setStyleVersion(version) { - this._styleVersion = version === undefined ? undefined : new Map(version); - } - #attributeResolver() { - const metasAt = this._attributeMetasResolver(); - return (element) => this.#attributeValues(metasAt(element)); - } - #attributeValues(metas) { - const cached = this.#attributeValuesCache.get(metas); - if (cached !== undefined) - return cached; - const attributes = new Map(); - for (const [key, meta] of metas) { - if (meta.value !== null) - attributes.set(key, meta.value); - } - this.#attributeValuesCache.set(metas, attributes); - return attributes; - } - _validateInsertPosition(position) { - const unicodePosition = this.convertPos(position, "utf16", "unicode"); - if (unicodePosition === undefined) { - throw new RangeError(`text position ${position} is out of range`); - } - return unicodePosition; - } - _unicodePosition(position) { - const unicodePosition = this.convertPos(position, "utf16", "unicode"); - if (unicodePosition === undefined) { - throw new RangeError(`text position ${position} is not on a UTF-16 boundary`); - } - return unicodePosition; - } - _reset() { - this._sequence.reset(); - this.#validatedSnapshotSpans = undefined; - this._detachedCounter = 0; - this._detachedStyleCounter = 0; - this._attributeHistoryComplete = true; - this._styleIndex.reset(); - this._styleVersion = undefined; - } -} -export class LoroMovableList extends LoroList { - _valueHistoryComplete = true; - _moveHistoryComplete = true; - kind() { - return "MovableList"; - } - move(from, to) { - validateIndex(from, this.length); - if (!Number.isSafeInteger(to) || to < 0 || to >= this.length) { - throw new RangeError(`movable-list destination ${to} is out of range`); - } - if (from === to) - return; - if (this._doc === undefined) { - this._applyMove(from, to); - return; - } - this._doc._movableMove(this, from, to); - } - mov(from, to) { - this.move(from, to); - } - set(pos, value) { - validateIndex(pos, this.length); - if (isContainer(value)) - throw new TypeError("use setContainer() to attach a child container"); - if (this._doc === undefined) { - this._sequence.atVisible(pos).value = normalizeDetachedValue(value); - return; - } - this._doc._movableSet(this, pos, value); - } - setContainer(pos, child) { - validateIndex(pos, this.length); - if (this._doc === undefined) { - const element = this._sequence.atVisible(pos); - element.value = child; - this._bindChildren([element]); - return child; - } - return this._doc._movableSetContainer(this, pos, child); - } - getCreatorAt(pos) { - return this._sequence.atVisible(pos)?.id.peer.toString(); - } - getLastMoverAt(pos) { - return this._sequence.atVisible(pos)?.id.peer.toString(); - } - getLastEditorAt(pos) { - return this._sequence.atVisible(pos)?.id.peer.toString(); - } - _applyMove(from, to, operation, replaceExisting = false) { - const element = this._sequence.atVisible(from); - if (element === undefined) - return; - const beforePrevious = this._sequence.previousVisible(element)?.id; - const beforeNext = this._sequence.nextVisible(element)?.id; - this._sequence.moveVisible(from, to); - if (operation === undefined) - return; - const meta = { - id: operation.id, - lamport: operation.lamport, - beforePrevious, - beforeNext, - afterPrevious: this._sequence.previousVisible(element)?.id, - afterNext: this._sequence.nextVisible(element)?.id, - }; - let history = element.moveHistory; - if (history === undefined) { - history = []; - element.moveHistory = history; - } - const index = lowerBoundSequenceMoveMeta(history, meta); - const existing = history[index]; - if (existing !== undefined && - existing.id.peer === meta.id.peer && - existing.id.counter === meta.id.counter) { - if (replaceExisting) - history[index] = meta; - } - else { - history.splice(index, 0, meta); - } - } - _moveToAnchors(element, previousId, nextId) { - if (element.deleted) - return; - if (nextId === undefined) { - if (this._sequence.nextVisible(element) !== undefined) { - this._sequence.moveBefore(element, undefined); - } - return; - } - const next = this._sequence.findById(nextId); - if (next !== undefined && !next.deleted && next !== element) { - this._sequence.moveBefore(element, next); - return; - } - const previous = previousId === undefined ? undefined : this._sequence.findById(previousId); - if (previous !== undefined && !previous.deleted && previous !== element) { - const successor = this._sequence.nextVisible(previous); - if (successor !== element) - this._sequence.moveBefore(element, successor); - return; - } - if (previousId === undefined) { - const first = this._sequence.atVisible(0); - if (first !== undefined && first !== element) { - this._sequence.moveBefore(element, first); - } - } - } - _applySet(element, value, meta) { - if (meta === undefined) { - element.value = value; - this._bindChildren([element]); - return; - } - let history = element.valueHistory; - if (history === undefined) { - history = []; - element.valueHistory = history; - } - const index = lowerBoundSequenceValueMeta(history, meta); - const existing = history[index]; - if (existing === undefined || - existing.id.peer !== meta.id.peer || - existing.id.counter !== meta.id.counter) { - history.splice(index, 0, meta); - } - const winner = history.at(-1); - element.value = winner.value; - this._bindChildren([element]); - } - _reset() { - super._reset(); - this._valueHistoryComplete = true; - this._moveHistoryComplete = true; - } -} -export class LoroCounter extends LoroContainer { - _value = 0; - kind() { - return "Counter"; - } - increment(value) { - if (!Number.isFinite(value)) - throw new TypeError("counter increment must be finite"); - if (this._doc === undefined) { - this._value += value; - return; - } - this._doc._counterIncrement(this, value); - } - decrement(value) { - this.increment(-value); - } - get value() { - return this._value; - } - getValue() { - return this._value; - } - toJSON() { - return this._value; - } - getShallowValue() { - return this._value; - } - _reset() { - this._value = 0; - } -} -const DELETED_TREE_ROOT_ID = { - peer: 18446744073709551615n, - counter: 2147483647, -}; -export class LoroTree extends LoroContainer { - _nodes = new Map(); - // Parent-keyed children indexes, keyed numerically (peer -> counter) so tree - // reads/writes never stringify ids. Root children live in `_rootChildren`. - _children = new Map(); - _rootChildren; - _deletedRoots; - _fractionalIndexEnabled = true; - kind() { - return "Tree"; - } - createNode(parent, index) { - if (this._doc === undefined) - throw new Error("tree nodes can only be created on an attached tree"); - return this._doc._treeCreate(this, parent, index); - } - move(target, parent, index) { - if (this._doc === undefined) - throw new Error("tree nodes can only be moved on an attached tree"); - this._doc._treeMove(this, target, parent, index); - } - delete(target) { - if (this._doc === undefined) - return; - this._doc._treeDelete(this, target); - } - has(target) { - return this._nodes.has(target); - } - contains(target) { - return this.has(target); - } - isNodeDeleted(target) { - const node = this._nodes.get(target); - if (node === undefined) { - throw new RangeError(`tree node ${target} does not exist`); - } - return this._isRecordDeleted(node); - } - enableFractionalIndex(jitter = 0) { - if (!Number.isSafeInteger(jitter) || jitter < 0 || jitter > 0xff) { - throw new RangeError("fractional-index jitter must be an unsigned byte"); - } - this._fractionalIndexEnabled = true; - } - disableFractionalIndex() { - this._fractionalIndexEnabled = false; - } - isFractionalIndexEnabled() { - return this._fractionalIndexEnabled; - } - getNodeByID(target) { - const record = this._nodes.get(target); - return record === undefined ? undefined : new LoroTreeNode(this, record.id); - } - getNodes(options = {}) { - const records = this._nodesUnder(this._childrenOf(undefined)); - if (options.withDeleted === true) { - records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); - } - return records.map((record) => new LoroTreeNode(this, record.id)); - } - _recordsForSnapshot() { - const records = this._nodesUnder(this._childrenOf(undefined)); - records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); - return records; - } - nodes() { - return this.getNodes(); - } - roots() { - return this._childrenOf(undefined).map((record) => new LoroTreeNode(this, record.id)); - } - toArray() { - return this._childrenOf(undefined).map((record, index) => this._recordToNodeValue(record, index)); - } - toJSON() { - return this._childrenOf(undefined).map((record, index) => this._recordToValue(record, index)); - } - getShallowValue() { - return this._childrenOf(undefined).map((record, index) => this._recordToShallowValue(record, index)); - } - _childrenOf(parent) { - return this._childrenIndexFor(parent)?.values() ?? []; - } - _nodeAt(parent, index) { - const record = this._childrenIndexFor(parent)?.at(index); - return record === undefined ? undefined : new LoroTreeNode(this, record.id); - } - _rootJsonValueAt(index) { - const roots = this._childrenIndexFor(undefined); - const normalized = index < 0 ? (roots?.size ?? 0) + index : index; - const record = roots?.at(normalized); - return record === undefined - ? undefined - : this._recordToValue(record, normalized); - } - _rootCount() { - return this._childrenIndexFor(undefined)?.size ?? 0; - } - _rootJsonValuesRange(start, end) { - return (this._childrenIndexFor(undefined)?.valuesRange(start, end) ?? []).map((record, offset) => this._recordToValue(record, start + offset)); - } - _positionFor(parent, index, exclude) { - const children = this._childrenIndexFor(parent); - const excluded = exclude === undefined ? undefined : this._nodes.get(formatTreeId(exclude)); - const excludedIndex = excluded === undefined || - excluded.deleted || - !sameOptionalId(excluded.parent, parent) - ? undefined - : children?.indexOf(excluded); - const length = (children?.size ?? 0) - (excludedIndex === undefined ? 0 : 1); - const position = index ?? length; - if (!Number.isSafeInteger(position) || position < 0 || position > length) { - throw new RangeError(`tree index ${position} is out of range`); - } - // The sibling lookups below skip the slot the excluded record still - // occupies; inlined to avoid allocating a closure per call. - let before; - if (position > 0) { - const siblingIndex = excludedIndex !== undefined && position - 1 >= excludedIndex - ? position - : position - 1; - before = children.at(siblingIndex).position; - } - let after; - if (position < length) { - const siblingIndex = excludedIndex !== undefined && position >= excludedIndex - ? position + 1 - : position; - after = children.at(siblingIndex).position; - } - return fractionalIndexBetween(before, after); - } - _setRecord(record) { - this._nodes.set(formatTreeId(record.id), record); - record.data._setParentBinding(this, { kind: "tree", record }); - if (record.deleted) { - record.parent = DELETED_TREE_ROOT_ID; - this._deletedRootsIndex().add(record); - } - else { - this._childrenIndex(record.parent).add(record); - } - } - _updateRecord(record, parent, position, writer, lastMoveId) { - if (record.deleted) { - this._deletedRoots?.delete(record); - } - else { - this._childrenIndexFor(record.parent)?.delete(record); - } - record.parent = parent; - record.position = position; - record.deleted = false; - record.writer = writer; - record.lastMoveId = lastMoveId; - this._childrenIndex(parent).add(record); - } - _deleteRecord(record, writer, lastMoveId) { - if (record.deleted) { - this._deletedRoots?.delete(record); - } - else { - this._childrenIndexFor(record.parent)?.delete(record); - } - record.parent = DELETED_TREE_ROOT_ID; - record.position = Uint8Array.of(0x80); - record.deleted = true; - record.writer = writer; - record.lastMoveId = lastMoveId; - this._deletedRootsIndex().add(record); - } - _removeRecord(record) { - if (record.deleted) { - this._deletedRoots?.delete(record); - } - else { - this._childrenIndexFor(record.parent)?.delete(record); - } - record.deleted = true; - this._nodes.delete(formatTreeId(record.id)); - } - _indexOf(record) { - if (record.deleted) - return -1; - return this._childrenIndexFor(record.parent)?.indexOf(record) ?? -1; - } - _isRecordDeleted(record) { - const visited = new Set(); - let current = record; - while (current !== undefined) { - if (current.deleted) - return true; - if (current.parent === undefined) - return false; - const key = formatTreeId(current.parent); - if (visited.has(key)) - return true; - visited.add(key); - current = this._nodes.get(key); - } - return true; - } - _recordToValue(record, index = this._indexOf(record)) { - return { - id: formatTreeId(record.id), - parent: record.parent === undefined ? null : formatTreeId(record.parent), - index, - fractional_index: bytesToHex(record.position).toUpperCase(), - meta: record.data.toJSON(), - children: this._childrenOf(record.id).map((child, childIndex) => this._recordToValue(child, childIndex)), - }; - } - _recordToNodeValue(record, index = this._indexOf(record)) { - return { - id: formatTreeId(record.id), - parent: record.deleted - ? "2147483647@18446744073709551615" - : record.parent === undefined - ? undefined - : formatTreeId(record.parent), - index: record.deleted ? 0 : index, - fractionalIndex: bytesToHex(record.position).toUpperCase() || "80", - meta: record.data.toJSON(), - children: this._childrenOf(record.id).map((child, childIndex) => this._recordToNodeValue(child, childIndex)), - }; - } - _recordToShallowValue(record, index = this._indexOf(record)) { - return { - id: formatTreeId(record.id), - parent: record.deleted - ? "2147483647@18446744073709551615" - : record.parent === undefined - ? null - : formatTreeId(record.parent), - index: record.deleted ? 0 : index, - fractional_index: bytesToHex(record.position).toUpperCase(), - meta: record.data.id, - children: this._childrenOf(record.id).map((child, childIndex) => this._recordToShallowValue(child, childIndex)), - }; - } - _reset() { - this._nodes.clear(); - this._children.clear(); - this._rootChildren = undefined; - this._deletedRoots = undefined; - } - _nodesUnder(roots) { - const output = []; - const queue = roots.slice(); - for (let index = 0; index < queue.length; index += 1) { - const record = queue[index]; - output.push(record); - queue.push(...this._childrenOf(record.id)); - } - return output; - } - _childrenIndexFor(parent) { - return parent === undefined - ? this._rootChildren - : this._children.get(parent.peer)?.get(parent.counter); - } - _childrenIndex(parent) { - if (parent === undefined) { - return (this._rootChildren ??= new OrderedIndex(compareTreeRecords)); - } - let byCounter = this._children.get(parent.peer); - if (byCounter === undefined) { - byCounter = new Map(); - this._children.set(parent.peer, byCounter); - } - let index = byCounter.get(parent.counter); - if (index === undefined) { - index = new OrderedIndex(compareTreeRecords); - byCounter.set(parent.counter, index); - } - return index; - } - _deletedRootsIndex() { - return (this._deletedRoots ??= new OrderedIndex(compareTreeRecords)); - } -} -export class LoroTreeNode { - #tree; - #id; - constructor(tree, id) { - this.#tree = tree; - this.#id = id; - } - free() { } - get id() { - return formatTreeId(this.#id); - } - get data() { - return this.#record().data; - } - createNode(index) { - return this.#tree.createNode(this.id, index); - } - move(parent, index) { - this.#tree.move(this.id, parent?.id, index); - } - moveAfter(target) { - const parent = target.parent(); - const index = target.index(); - if (index === undefined) - throw new RangeError(`tree node ${target.id} is deleted`); - this.move(parent, index + 1); - } - moveBefore(target) { - const index = target.index(); - if (index === undefined) - throw new RangeError(`tree node ${target.id} is deleted`); - this.move(target.parent(), index); - } - parent() { - const parent = this.#record().parent; - return parent === undefined ? undefined : new LoroTreeNode(this.#tree, parent); - } - children() { - return this.#tree - ._childrenOf(this.#id) - .map((record) => new LoroTreeNode(this.#tree, record.id)); - } - _childAt(index) { - return this.#tree._nodeAt(this.#id, index); - } - index() { - const record = this.#record(); - const index = this.#tree._indexOf(record); - return index < 0 ? undefined : index; - } - fractionalIndex() { - const record = this.#record(); - return this.#tree._fractionalIndexEnabled && !record.deleted - ? bytesToHex(record.position).toUpperCase() - : undefined; - } - isDeleted() { - return this.#tree._isRecordDeleted(this.#record()); - } - getLastMoveId() { - const id = this.#record().lastMoveId; - return { peer: id.peer.toString(), counter: id.counter }; - } - creationId() { - return { peer: this.#id.peer.toString(), counter: this.#id.counter }; - } - creator() { - return this.#id.peer.toString(); - } - toJSON() { - return this.#tree._recordToNodeValue(this.#record()); - } - _codecId() { - return this.#id; - } - #record() { - const record = this.#tree._nodes.get(this.id); - if (record === undefined) - throw new Error(`tree node ${this.id} does not exist`); - return record; - } -} -export class Cursor { - #containerId; - #id; - #side; - #originPosition; - constructor(containerId, id, side, originPosition = 0) { - this.#containerId = containerId; - this.#id = id; - this.#side = side; - this.#originPosition = originPosition; - } - free() { } - containerId() { - return this.#containerId; - } - pos() { - return this.#id === undefined - ? undefined - : { peer: this.#id.peer.toString(), counter: this.#id.counter }; - } - side() { - return this.#side; - } - kind() { - return "Cursor"; - } - encode() { - const writer = new PostcardWriter(); - writer.writeOption(this.#id, (output, id) => { - output.writeU64(id.peer); - output.writeI32(id.counter); - }); - writePostcardCursorContainer(writer, parseContainerId(this.#containerId)); - writer.writeU32(cursorSideToVariant(this.#side)); - writer.writeUsize(this.#originPosition); - return writer.toUint8Array(); - } - static decode(bytes) { - const reader = new PostcardReader(bytes); - const id = reader.readOption((input) => ({ - peer: input.readU64(), - counter: input.readI32(), - })); - const containerId = readPostcardCursorContainer(reader); - const side = cursorSideFromVariant(reader.readU32()); - const originPosition = reader.readUsize(); - reader.assertEnd(); - return new Cursor(formatContainerId(containerId), id, side, originPosition); - } - _idValue() { - return this.#id; - } - _originPositionValue() { - return this.#originPosition; - } -} -function writePostcardCursorContainer(writer, id) { - if (id.kind === "root") { - writer.writeU32(0); - writer.writeString(id.name); - } - else { - writer.writeU32(1); - writer.writeU64(id.peer); - writer.writeI32(id.counter); - } - writer.writeU8(containerTypeToHistoricalByte(id.containerType)); -} -function readPostcardCursorContainer(reader) { - const variant = reader.readU32(); - if (variant === 0) { - const name = reader.readString(); - const containerType = containerTypeFromHistoricalByte(reader.readU8()); - return { kind: "root", name, containerType }; - } - if (variant === 1) { - const peer = reader.readU64(); - const counter = reader.readI32(); - const containerType = containerTypeFromHistoricalByte(reader.readU8()); - return { kind: "normal", peer, counter, containerType }; - } - throw new TypeError(`unsupported cursor container variant ${variant}`); -} -function cursorSideToVariant(side) { - return side === -1 ? 0 : side === 0 ? 1 : 2; -} -function cursorSideFromVariant(variant) { - if (variant === 0) - return -1; - if (variant === 1) - return 0; - if (variant === 2) - return 1; - throw new TypeError(`unsupported cursor side variant ${variant}`); -} -export function isContainer(value) { - return value instanceof LoroContainer; -} -export function getType(value) { - return isContainer(value) ? value.kind() : "Json"; -} -export function runtimeValueToJson(value) { - if (isContainer(value)) - return value.toJSON(); - if (value instanceof Uint8Array) - return value.slice(); - if (Array.isArray(value)) - return value.map(runtimeValueToJson); - if (typeof value === "object" && value !== null) { - const record = value; - const output = {}; - for (const key in record) - output[key] = runtimeValueToJson(record[key]); - return output; - } - return value; -} -export function runtimeValueToShallow(value) { - if (isContainer(value)) - return value.id; - if (value instanceof Uint8Array) - return value.slice(); - if (Array.isArray(value)) - return value.map(runtimeValueToShallow); - if (typeof value === "object" && value !== null) { - const record = value; - const output = {}; - for (const key in record) - output[key] = runtimeValueToShallow(record[key]); - return output; - } - return value; -} -export function containerDeepValueWithId(container) { - return { cid: container.id, value: containerValueWithId(container) }; -} -function containerValueWithId(container) { - if (container instanceof LoroMap) { - return Object.fromEntries(container - .keys() - .map((key) => [key, runtimeValueDeepWithId(container._entries.get(key).value)])); - } - if (container instanceof LoroText) - return container.toString(); - if (container instanceof LoroCounter) - return container.value; - if (container instanceof LoroTree) { - const visit = (record) => { - return { - id: formatTreeId(record.id), - parent: record.parent === undefined ? null : formatTreeId(record.parent), - index: container._indexOf(record), - fractional_index: bytesToHex(record.position).toUpperCase(), - meta: containerDeepValueWithId(record.data), - children: container._childrenOf(record.id).map(visit), - }; - }; - return container._childrenOf(undefined).map(visit); - } - return container - ._visibleElements() - .map((element) => runtimeValueDeepWithId(element.value)); -} -function runtimeValueDeepWithId(value) { - if (isContainer(value)) - return containerDeepValueWithId(value); - if (value instanceof Uint8Array) - return value.slice(); - if (Array.isArray(value)) - return value.map(runtimeValueDeepWithId); - if (typeof value === "object" && value !== null) { - const record = value; - const output = {}; - for (const key in record) - output[key] = runtimeValueDeepWithId(record[key]); - return output; - } - return value; -} -export function cloneRuntimeValue(value) { - if (value === undefined) - return undefined; - if (isContainer(value)) - return value; - if (value instanceof Uint8Array) - return value.slice(); - if (Array.isArray(value)) - return value.map((item) => cloneRuntimeValue(item)); - if (typeof value === "object" && value !== null) { - const record = value; - const output = {}; - for (const key in record) - output[key] = cloneRuntimeValue(record[key]); - return output; - } - return value; -} -function normalizeDetachedValue(value) { - if (value === undefined) - return null; - if (value === null || typeof value === "string" || typeof value === "boolean") - return value; - if (typeof value === "number") { - if (!Number.isFinite(value)) - throw new TypeError("Loro values must be finite numbers"); - return value; - } - if (value instanceof Uint8Array) - return value.slice(); - if (Array.isArray(value)) - return value.map(normalizeDetachedValue); - if (isContainer(value)) - return value; - if (typeof value === "object") { - if (Object.getOwnPropertySymbols(value).length > 0) { - throw new TypeError("Object keys must be strings"); - } - const output = {}; - for (const key of Object.keys(value)) { - output[key] = normalizeDetachedValue(value[key]); - } - return output; - } - throw new TypeError(`unsupported Loro value type: ${typeof value}`); -} -function insertFugueElements(sequence, position, inserted, causalVersion, useOriginIndex = true) { - if (inserted.length === 0) - return; - const insertion = fugueInsertion(sequence, position, inserted[0].id, causalVersion, useOriginIndex); - const { insertIndex, originLeft, originRight } = insertion; - for (let index = 0; index < inserted.length; index += 1) { - inserted[index].originLeft = index === 0 ? originLeft : inserted[index - 1].id; - inserted[index].originRight = originRight; - } - sequence.insertAtPhysical(insertIndex, inserted); - recordFugueInsertion(sequence, insertion, inserted.map((element) => element.id)); -} -const fugueOriginIndexes = new WeakMap(); -// Callers consume a fugueInsertion result before the next insertion, so every -// path reuses this single fixed-shape record instead of allocating per edit. -const fugueInsertionResult = { - insertIndex: 0, - originLeft: undefined, - originRight: undefined, - indexUpdate: undefined, -}; -const fugueScanVisited = new Map(); -const fugueCandidateEntries = []; -const fugueCandidateIndexes = []; -const fugueCandidateOrder = []; -const fugueImplicitEntry = { id: { peer: 0n, counter: 0 }, originLeft: undefined, originRight: undefined }; -function compareFugueCandidateOrder(left, right) { - return fugueCandidateIndexes[left] - fugueCandidateIndexes[right]; -} -function fugueInsertion(sequence, position, insertedId, causalVersion, useOriginIndex = true, positionHint) { - const current = positionHint?.current ?? sequence.isFullyIncluded(causalVersion); - const currentContext = positionHint === undefined && current - ? sequence.visibleInsertionContext(Math.min(position, sequence.visibleLength)) - : undefined; - const authoredVisible = positionHint === undefined && !current - ? sequence.causalView(causalVersion) - : undefined; - const authoredLength = current ? sequence.visibleLength : authoredVisible?.length; - const authoredPosition = Math.min(position, authoredLength ?? position); - const left = positionHint === undefined - ? current - ? currentContext?.left - : authoredVisible?.at(authoredPosition - 1) - : positionHint.left; - const originLeft = left?.id; - const startIndex = positionHint?.startIndex ?? - currentContext?.startIndex ?? - (left === undefined ? 0 : (sequence.physicalIndexOf(left) ?? -1) + 1); - const currentRight = !current - ? undefined - : positionHint?.startIndex !== undefined - ? positionHint.right - : currentContext !== undefined - ? currentContext.right - : sequence.atPhysicalRaw(startIndex); - const right = current - ? currentRight === undefined - ? undefined - : { element: currentRight, index: startIndex } - : sequence.findNextIncludedPhysical(startIndex, causalVersion); - const originRightIndex = right?.index ?? sequence.allLength; - const originRight = right?.element.id; - // There is no concurrent/future interval to order. Avoid allocating and - // maintaining an origin index for the overwhelmingly common local-edit path; - // structureVersion will force a rebuild if a later merge needs the index. - if (startIndex === originRightIndex) { - fugueInsertionResult.insertIndex = startIndex; - fugueInsertionResult.originLeft = originLeft; - fugueInsertionResult.originRight = originRight; - fugueInsertionResult.indexUpdate = undefined; - return fugueInsertionResult; - } - if (useOriginIndex) { - const indexed = indexedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId); - if (indexed !== undefined) - return indexed; - } - return scannedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId); -} -function scannedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId) { - const parentRightIndex = directRightParentIndex(sequence, originLeft, originRight); - let insertIndex = startIndex; - let scanning = false; - fugueScanVisited.clear(); - for (let index = startIndex; index < originRightIndex; index += 1) { - const other = sequence.atPhysicalRaw(index); - if (!sameOptionalId(other.originLeft, originLeft) && - (other.originLeft === undefined || - fugueScanVisited.get(other.originLeft.peer)?.has(other.originLeft.counter) !== - true)) { - break; - } - let visitedCounters = fugueScanVisited.get(other.id.peer); - if (visitedCounters === undefined) { - visitedCounters = new Set(); - fugueScanVisited.set(other.id.peer, visitedCounters); - } - visitedCounters.add(other.id.counter); - if (sameOptionalId(other.originLeft, originLeft)) { - if (sameOptionalId(other.originRight, originRight)) { - if (other.id.peer > insertedId.peer) - break; - scanning = false; - } - else { - const otherParentRightIndex = directRightParentIndex(sequence, originLeft, other.originRight); - const ordering = compareOptionalPositions(otherParentRightIndex, parentRightIndex); - if (ordering < 0) - scanning = true; - else if (ordering === 0 && other.id.peer > insertedId.peer) - break; - else - scanning = false; - } - } - if (!scanning) - insertIndex = index + 1; - } - fugueInsertionResult.insertIndex = insertIndex; - fugueInsertionResult.originLeft = originLeft; - fugueInsertionResult.originRight = originRight; - fugueInsertionResult.indexUpdate = undefined; - return fugueInsertionResult; -} -function indexedFugueInsertion(sequence, startIndex, originRightIndex, originLeft, originRight, insertedId) { - const originIndex = getFugueOriginIndex(sequence); - // Reused candidate columns: gather the direct children inside the scanned - // interval without per-call wrapper, slice, or filtered-copy allocations. - fugueCandidateEntries.length = 0; - fugueCandidateIndexes.length = 0; - const explicit = originIndex.explicitChildren.get(optionalSequenceIdKey(originLeft)); - if (explicit !== undefined) { - for (const entry of explicit) { - const element = sequence.findByIdRaw(entry.id); - const index = element === undefined ? undefined : sequence.physicalIndexOf(element); - if (index === undefined) - return undefined; - if (index >= startIndex && index < originRightIndex) { - fugueCandidateEntries.push(entry); - fugueCandidateIndexes.push(index); - } - } - } - if (originLeft !== undefined) { - // Consecutive IDs keep their single-child edge implicit; derive it on - // lookup instead of recording an entry for every consecutive insert. - const implicit = sequence.findByIdRaw({ - peer: originLeft.peer, - counter: originLeft.counter + 1, - }); - if (implicit !== undefined && sameOptionalId(implicit.originLeft, originLeft)) { - const index = sequence.physicalIndexOf(implicit); - if (index === undefined) - return undefined; - if (index >= startIndex && index < originRightIndex) { - fugueImplicitEntry.id = implicit.id; - fugueImplicitEntry.originLeft = originLeft; - fugueImplicitEntry.originRight = implicit.originRight; - fugueCandidateEntries.push(fugueImplicitEntry); - fugueCandidateIndexes.push(index); - } - } - } - const count = fugueCandidateIndexes.length; - fugueCandidateOrder.length = count; - for (let index = 0; index < count; index += 1) - fugueCandidateOrder[index] = index; - fugueCandidateOrder.sort(compareFugueCandidateOrder); - const parentRightIndex = directRightParentIndex(sequence, originLeft, originRight); - let insertIndex = startIndex; - let scanning = false; - for (let orderIndex = 0; orderIndex < count; orderIndex += 1) { - const candidate = fugueCandidateOrder[orderIndex]; - const other = fugueCandidateEntries[candidate]; - if (sameOptionalId(other.originRight, originRight)) { - if (other.id.peer > insertedId.peer) - break; - scanning = false; - } - else { - const otherParentRightIndex = directRightParentIndex(sequence, originLeft, other.originRight); - const ordering = compareOptionalPositions(otherParentRightIndex, parentRightIndex); - if (ordering < 0) - scanning = true; - else if (ordering === 0 && other.id.peer > insertedId.peer) - break; - else - scanning = false; - } - if (!scanning) { - insertIndex = - orderIndex + 1 < count - ? fugueCandidateIndexes[fugueCandidateOrder[orderIndex + 1]] - : originRightIndex; - } - } - fugueInsertionResult.insertIndex = insertIndex; - fugueInsertionResult.originLeft = originLeft; - fugueInsertionResult.originRight = originRight; - fugueInsertionResult.indexUpdate = originIndex; - return fugueInsertionResult; -} -function getFugueOriginIndex(sequence) { - const cached = fugueOriginIndexes.get(sequence); - if (cached?.structureVersion === sequence.structureVersion) - return cached; - // Consecutive IDs form the common single-child Fugue chain, so derive those - // edges on lookup instead of allocating a map entry for every text scalar. - // Any untracked physical edit changes structureVersion and rebuilds the - // explicit branch index before it is queried again. - const rebuilt = { - structureVersion: sequence.structureVersion, - explicitChildren: new Map(), - }; - sequence.forEachPhysicalRaw((element) => { - recordFugueOriginEntry(rebuilt, element.id, element.originLeft, element.originRight); - }); - fugueOriginIndexes.set(sequence, rebuilt); - return rebuilt; -} -function recordFugueInsertion(sequence, insertion, insertedIds) { - const index = insertion.indexUpdate; - if (index === undefined) - return; - let originLeft = insertion.originLeft; - for (const id of insertedIds) { - recordFugueOriginEntry(index, id, originLeft, insertion.originRight); - originLeft = id; - } - index.structureVersion = sequence.structureVersion; -} -function recordFugueInsertionRun(sequence, insertion, startId) { - const index = insertion.indexUpdate; - if (index === undefined) - return; - recordFugueOriginEntry(index, startId, insertion.originLeft, insertion.originRight); - index.structureVersion = sequence.structureVersion; -} -function recordFugueOriginEntry(index, id, originLeft, originRight) { - if (originLeft !== undefined && - id.peer === originLeft.peer && - id.counter === originLeft.counter + 1) { - return; - } - const key = optionalSequenceIdKey(originLeft); - const children = index.explicitChildren.get(key); - const entry = { - id: { peer: id.peer, counter: id.counter }, - originLeft: originLeft === undefined - ? undefined - : { peer: originLeft.peer, counter: originLeft.counter }, - originRight: originRight === undefined - ? undefined - : { peer: originRight.peer, counter: originRight.counter }, - }; - if (children === undefined) - index.explicitChildren.set(key, [entry]); - else - children.push(entry); -} -function directRightParentIndex(sequence, originLeft, originRight) { - if (originRight === undefined) - return undefined; - const element = sequence.findByIdRaw(originRight); - if (element === undefined) - return undefined; - const index = sequence.physicalIndexOf(element); - return index !== undefined && sameOptionalId(element.originLeft, originLeft) - ? index - : undefined; -} -function compareOptionalPositions(left, right) { - if (left !== undefined && right !== undefined) - return left - right; - if (left !== undefined) - return -1; - if (right !== undefined) - return 1; - return 0; -} -function sequenceIdKey(id) { - return `${id.peer}:${id.counter}`; -} -function optionalSequenceIdKey(id) { - return id === undefined ? "root" : `id:${sequenceIdKey(id)}`; -} -function compareWriters(left, right) { - return (left.lamport - right.lamport || - (left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0)); -} -function compareSequenceValueMeta(left, right) { - return compareWriters({ peer: left.id.peer, lamport: left.lamport }, { peer: right.id.peer, lamport: right.lamport }); -} -function lowerBoundSequenceValueMeta(history, meta) { - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (compareSequenceValueMeta(history[middle], meta) < 0) - low = middle + 1; - else - high = middle; - } - return low; -} -function lowerBoundSequenceMoveMeta(history, meta) { - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - const current = history[middle]; - const order = compareWriters({ peer: current.id.peer, lamport: current.lamport }, { peer: meta.id.peer, lamport: meta.lamport }); - if (order < 0) - low = middle + 1; - else - high = middle; - } - return low; -} -function validateRange(position, length, total) { - if (!Number.isSafeInteger(position) || - !Number.isSafeInteger(length) || - position < 0 || - length < 0 || - position + length > total) { - throw new RangeError(`range ${position}..${position + length} is out of bounds for length ${total}`); - } -} -function validateIndex(index, length) { - if (!Number.isSafeInteger(index) || index < 0 || index >= length) { - throw new RangeError(`index ${index} is out of range for length ${length}`); - } -} -function sameOptionalId(left, right) { - return left === undefined || right === undefined - ? left === right - : left.peer === right.peer && left.counter === right.counter; -} -function comparePositions(left, right) { - const length = Math.min(left.length, right.length); - for (let index = 0; index < length; index += 1) { - if (left[index] !== right[index]) - return left[index] - right[index]; - } - return left.length - right.length; -} -function compareTreeRecords(left, right) { - return (comparePositions(left.position, right.position) || - compareWriters(left.writer, right.writer) || - codecIdsCompare(left.id, right.id)); -} -function codecIdsCompare(left, right) { - return left.peer < right.peer - ? -1 - : left.peer > right.peer - ? 1 - : left.counter - right.counter; -} -function textElementsToDelta(elements, attributesAt) { - const output = []; - let values = []; - let attributes; - const flush = () => { - if (values.length === 0) - return; - const insert = values.join(""); - values = []; - let jsonAttributes; - if (attributes !== undefined) { - for (const [key, value] of attributes) { - (jsonAttributes ??= {})[key] = runtimeValueToJson(value); - } - } - if (jsonAttributes === undefined) - output.push({ insert }); - else - output.push({ insert, attributes: jsonAttributes }); - }; - for (const element of elements) { - const nextAttributes = attributesAt(element); - if (values.length > 0 && !textAttributesEqual(attributes, nextAttributes)) { - flush(); - } - attributes = nextAttributes; - values.push(element.value); - } - flush(); - return output; -} -function textAttributesEqual(left, right) { - if (left === right) - return true; - if ((left?.size ?? 0) !== (right?.size ?? 0)) - return false; - for (const [key, value] of left ?? []) { - if (right?.has(key) !== true || right.get(key) !== value) - return false; - } - return true; -} -function isTextPosType(value) { - return value === "unicode" || value === "utf16" || value === "utf8"; -} -function utf8CodePointLength(value) { - const codePoint = value.codePointAt(0) ?? 0; - if (codePoint <= 0x7f) - return 1; - if (codePoint <= 0x7ff) - return 2; - if (codePoint <= 0xffff) - return 3; - return 4; -} -/** The code point whose UTF-16 end offset is `end`, ignoring pairs opened before `lowerBound`. */ -function codePointBefore(text, end, lowerBound) { - const unit = text.charCodeAt(end - 1); - if (unit >= 0xdc00 && unit <= 0xdfff && end - 2 >= lowerBound) { - const high = text.charCodeAt(end - 2); - if (high >= 0xd800 && high <= 0xdbff) { - return 65536 + ((high - 0xd800) << 10) + (unit - 0xdc00); - } - } - return unit; -} -function updateText(container, target) { - const current = container.toString(); - const currentLength = current.length; - const targetLength = target.length; - // Common prefix/suffix measured in UTF-16 offsets while comparing by code - // point, so astral characters are never split and no arrays are built. - let prefixLength = 0; - while (prefixLength < currentLength && prefixLength < targetLength) { - const codePoint = current.codePointAt(prefixLength); - if (codePoint !== target.codePointAt(prefixLength)) - break; - prefixLength += codePoint > 0xffff ? 2 : 1; - } - let suffixLength = 0; - while (suffixLength < currentLength - prefixLength && - suffixLength < targetLength - prefixLength) { - const codePoint = codePointBefore(current, currentLength - suffixLength, prefixLength); - if (codePoint !== codePointBefore(target, targetLength - suffixLength, prefixLength)) { - break; - } - suffixLength += codePoint > 0xffff ? 2 : 1; - } - container.delete(prefixLength, currentLength - prefixLength - suffixLength); - container.insert(prefixLength, target.slice(prefixLength, targetLength - suffixLength)); -} diff --git a/loro-js/src/runtime/document.js b/loro-js/src/runtime/document.js deleted file mode 100644 index 6281c1fe3..000000000 --- a/loro-js/src/runtime/document.js +++ /dev/null @@ -1,6813 +0,0 @@ -import packageMetadata from "../../package.json" with { type: "json" }; -import { bytesEqual, bytesToHex, hexToBytes } from "../codec/bytes"; -import { decodeChangeBlock, encodeSingleChangeBlock, validateChangeBlock, } from "../codec/change-block-codec"; -import { containerTypeFromRawByte, containerTypeToRawByte } from "../codec/container-id"; -import { decodeDocument, decodeFastSnapshotBody, decodeFastUpdatesBody, encodeDocument, encodeFastSnapshotBody, encodeFastUpdatesBody, EncodeMode, } from "../codec/document"; -import { decodeChangeBlockKey, encodeChangeBlockKey } from "../codec/id"; -import { decodeSstable, encodeSstable } from "../codec/sstable"; -import { decodeStateSnapshotStore, encodeStateSnapshotStore, } from "../codec/state-snapshot"; -import { ContainerType as CodecContainerType, } from "../codec/types"; -import { decodePostcardFrontiers, decodePostcardVersionVector, encodePostcardFrontiers, encodePostcardVersionVector, } from "../codec/version"; -import { cloneRuntimeValue, containerDeepValueWithId, Cursor, isContainer, LoroCounter, LoroList, LoroMap, LoroMovableList, LoroText, LoroTree, LoroTreeNode, runtimeValueToJson, } from "./containers"; -import { SequenceEventDiff } from "./event-diff"; -import { codecTypeToPublic, containerIdsEqual, formatContainerId, formatOpId, formatTreeId, idsEqual, isContainerId, parseContainerId, parseOpId, parsePeerId, parseTreeId, peerIdToString, publicTypeToCodec, } from "./ids"; -import { compileJsonPathEventMatcher, evaluateJsonPath } from "./jsonpath"; -import { isMergeableContainerId, mergeableMarker, newMergeableContainerId, parseMergeableMarker, } from "./mergeable"; -import { OrderedIndex } from "./ordered-index"; -import { VersionVector } from "./version-vector"; -const VERSION_KEY = Uint8Array.of(0x76, 0x76); -const FRONTIERS_KEY = Uint8Array.of(0x66, 0x72); -const START_VERSION_KEY = Uint8Array.of(0x73, 0x76); -const START_FRONTIERS_KEY = Uint8Array.of(0x73, 0x66); -let fallbackPeer = 1n; -// Scratch buffer for Counter snapshot float64 <-> bits conversion. Used -// synchronously and never retained. -const float64BitsScratch = new DataView(new ArrayBuffer(8)); -// Shared item for MovableList snapshots; the state-snapshot encoder only -// reads item fields, so every slot can reference this frozen object. -const SHARED_MOVABLE_LIST_STATE_ITEM = Object.freeze({ - invisibleListItems: 0n, - positionIdEqualsElementId: true, - elementIdEqualsLastSetId: true, -}); -// Scratch single-run list reused for synchronous, non-retaining SequenceIndex -// queries (containsIdRuns / canShowIdRunsAt). Those callees read the runs -// synchronously and never store them, so reuse is safe on this -// single-threaded runtime. Never pass these objects to anything that retains -// them. -const scratchSequenceIdRun = { start: { peer: 0n, counter: 0 }, length: 0 }; -const scratchSequenceIdRuns = [scratchSequenceIdRun]; -export class LoroDoc { - #peer = generatePeerId(); - #history = new Map(); - #historyOrder = new OrderedIndex(compareHistoryRecords); - #historyByPeer = new Map(); - #historyEndByPeer = new Map(); - #historyOperationCount = 0; - #sortedHistoryCache; - #historyFrontiers = new Map(); - #dependencyVersionCache = new WeakMap(); - #mapOperationHistory = new Map(); - #treeOperationHistory = new Map(); - #movableOrderHistory = new Map(); - #movableMovePeers = new Map(); - #containersWithOperations = new Set(); - #containerKeys = new WeakMap(); - #pendingHistory = new Map(); - #deferredSnapshotHistory; - #containers = new Map(); - #roots = new Map(); - #pending; - #nextCounter = 0; - #recordTimestamp = false; - #changeMergeInterval = 1000n; - #nextCommitOptions = {}; - #subscribers = new Set(); - #localUpdateSubscribers = new Set(); - #containerSubscribers = new Map(); - #firstCommitSubscribers = new Set(); - #preCommitSubscribers = new Set(); - #seenCommittedPeers = new Set(); - #committing = false; - #preCommitRecord; - #detached = false; - #detachedEditing = false; - #checkoutVersion; - #hideEmptyRoots = false; - #shallowStartVersion = new VersionVector(); - #shallowRootVersion = new VersionVector(); - #shallowRootFrontiers = []; - #shallowRootStore; - #textStyles = new Map([ - ["bold", "after"], - ["italic", "after"], - ["underline", "after"], - ["link", "none"], - ["highlight", "none"], - ["comment", "none"], - ["code", "none"], - ]); - #defaultTextStyle; - static fromSnapshot(snapshot) { - const doc = new LoroDoc(); - doc.import(snapshot); - return doc; - } - free() { } - get peerId() { - return this.#peer; - } - get peerIdStr() { - return peerIdToString(this.#peer); - } - setPeerId(peer) { - if (this.#pending !== undefined) - throw new Error("cannot change peer id with pending operations"); - this.#peer = parsePeerId(peer); - this.#nextCounter = this.oplogVersion().get(this.#peer) ?? 0; - } - setRecordTimestamp(enabled) { - this.#recordTimestamp = enabled; - } - setChangeMergeInterval(interval) { - this.#changeMergeInterval = numberToI64(interval); - } - setDetachedEditing(enabled) { - this.#detachedEditing = enabled; - if (enabled && this.#detached) { - this.#commit({}, true); - this.#renewPeerId(); - } - } - isDetachedEditingEnabled() { - return this.#detachedEditing; - } - configTextStyle(styles) { - this.#textStyles.clear(); - for (const [key, style] of Object.entries(styles)) { - if (key.includes(":")) - throw new TypeError("text style keys cannot contain ':'"); - assertTextStyleExpand(style.expand); - this.#textStyles.set(key, style.expand); - } - } - configDefaultTextStyle(style) { - if (style === undefined) { - this.#defaultTextStyle = undefined; - return; - } - assertTextStyleExpand(style.expand); - this.#defaultTextStyle = style.expand; - } - getMap(name) { - return this.#getContainer(name, "Map"); - } - getList(name) { - return this.#getContainer(name, "List"); - } - getMovableList(name) { - return this.#getContainer(name, "MovableList"); - } - getText(name) { - return this.#getContainer(name, "Text"); - } - getTree(name) { - return this.#getContainer(name, "Tree"); - } - getCounter(name) { - return this.#getContainer(name, "Counter"); - } - getContainerById(id) { - const parsed = parseContainerId(id); - if (parsed.kind === "root" && !isMergeableContainerId(parsed)) { - return this.#getOrCreateContainer(parsed); - } - const container = this.#containers.get(formatContainerId(parsed)); - if (container !== undefined && - isMergeableContainerId(parsed) && - this._isContainerDeleted(container) && - !this.#containerHasOperations(parsed)) { - return undefined; - } - return container; - } - hasContainer(id) { - return this.getContainerById(id) !== undefined; - } - #containerHasOperations(id) { - this.#materializeDeferredHistory(); - return this.#containersWithOperations.has(formatContainerId(id)); - } - deleteRootContainer(id) { - const parsed = parseContainerId(id); - if (parsed.kind !== "root") - throw new TypeError("only root containers can be deleted directly"); - this.#roots.delete(parsed.name); - this.#containers.delete(formatContainerId(parsed)); - } - setHideEmptyRootContainers(hide) { - this.#hideEmptyRoots = hide; - } - commit(options = {}) { - this.#commit(options, false); - } - #commit(options, preserveOptionsOnEmpty) { - if (this.#committing) - return; - const pending = this.#pending; - if (pending === undefined || pending.operations.length === 0) { - if (!preserveOptionsOnEmpty) - this.#nextCommitOptions = {}; - return; - } - const mergedOptions = { - ...this.#nextCommitOptions, - ...options, - }; - this.#nextCommitOptions = {}; - const isFirstCommit = !this.#seenCommittedPeers.has(pending.id.peer); - this.#committing = true; - try { - if (isFirstCommit) { - this.#seenCommittedPeers.add(pending.id.peer); - for (const listener of this.#firstCommitSubscribers) { - listener({ peer: peerIdToString(pending.id.peer) }); - } - } - const latestTimestamp = this.#greatestTimestampAt(pending.dependencies); - if (this.#preCommitSubscribers.size > 0) { - const initialTimestamp = maxBigInt(latestTimestamp, numberToI64(mergedOptions.timestamp ?? (this.#recordTimestamp ? Date.now() / 1000 : 0))); - const provisional = { - id: pending.id, - dependencies: pending.dependencies, - lamport: pending.lamport, - timestamp: initialTimestamp, - message: mergedOptions.message, - operations: pending.operations, - }; - const modifier = new ChangeModifier(mergedOptions); - this.#preCommitRecord = { - change: provisional, - keys: pending.keys, - keyIndices: undefined, - }; - try { - for (const listener of this.#preCommitSubscribers) { - listener({ - changeMeta: publicChange(provisional), - origin: mergedOptions.origin ?? "", - modifier, - }); - } - } - finally { - this.#preCommitRecord = undefined; - } - } - const atomLength = pending.operationLength; - const timestamp = maxBigInt(latestTimestamp, numberToI64(mergedOptions.timestamp ?? (this.#recordTimestamp ? Date.now() / 1000 : 0))); - const change = { - id: pending.id, - dependencies: pending.dependencies, - lamport: pending.lamport, - timestamp, - message: mergedOptions.message, - operations: pending.operations, - }; - this.#pending = undefined; - const committedRecord = { - change, - keys: pending.keys, - keyIndices: undefined, - }; - const previous = this.#mergeablePreviousRecord(change, this.#changeMergeInterval); - let updateRecord = committedRecord; - if (previous === undefined) { - this.#setHistoryRecord(changeKey(change.id), committedRecord, committedRecord); - } - else { - const previousLength = changeLength(previous.change); - updateRecord = appendHistoryRecord(previous, committedRecord, previousLength); - this.#appendMergedHistoryRecord(previous, updateRecord, previousLength); - } - this.#nextCounter = change.id.counter + atomLength; - if (this.#detached) { - const checkoutVersion = this.#checkoutVersion ?? new VersionVector(); - checkoutVersion.set(change.id.peer, change.id.counter + atomLength); - this.#checkoutVersion = checkoutVersion; - } - if (this.#localUpdateSubscribers.size > 0) { - const bytes = this.#encodeUpdates([updateRecord]); - for (const listener of this.#localUpdateSubscribers) - listener(bytes.slice()); - } - if (this.#hasEventSubscribers()) { - this.#emit("local", mergedOptions.origin, pending.from, this.#frontiersCodec(), pending.changedContainers, pending.beforeValues, this.#recordedEventDiffs(pending)); - } - } - catch (error) { - if (isFirstCommit) - this.#seenCommittedPeers.delete(pending.id.peer); - throw error; - } - finally { - this.#committing = false; - } - } - getPendingTxnLength() { - return this.#pending?.operationLength ?? 0; - } - setNextCommitMessage(message) { - this.#nextCommitOptions = { ...this.#nextCommitOptions, message }; - } - setNextCommitOrigin(origin) { - this.#nextCommitOptions = { ...this.#nextCommitOptions, origin }; - } - setNextCommitTimestamp(timestamp) { - this.#nextCommitOptions = { ...this.#nextCommitOptions, timestamp }; - } - setNextCommitOptions(options) { - this.#nextCommitOptions = { ...options }; - } - clearNextCommitOptions() { - this.#nextCommitOptions = {}; - } - export(mode) { - this.#commit({}, true); - if (mode.mode === "update") { - const from = mode.from ?? new VersionVector(); - const effectiveFrom = from.clone(); - effectiveFrom.merge(this.#shallowStartVersion); - const records = this.#recordsInVersionRange(effectiveFrom, this.#historyVersion()); - return this.#encodeUpdates(records); - } - if (mode.mode === "updates-in-range") { - const records = this.#recordsInSpans(mode.spans); - return this.#encodeUpdates(records); - } - if (mode.mode === "shallow-snapshot") { - return this.#encodeShallowSnapshot(mode.frontiers); - } - return this.#encodeSnapshot(); - } - exportJsonUpdates(start, end, withPeerCompression = true) { - this.#commit({}, true); - const startVersion = new VersionVector(start); - const endVersion = end === undefined || end === null ? this.#historyVersion() : new VersionVector(end); - const records = this.#recordsInVersionRange(startVersion, endVersion); - return historyRecordsToJsonSchema(records, startVersion, withPeerCompression); - } - exportJsonInIdSpan(span) { - this.#commit({}, true); - const records = this.#recordsInSpans([ - { id: { peer: span.peer, counter: span.counter }, len: span.length }, - ]); - return historyRecordsToJsonSchema(records, new VersionVector(), false, false).changes; - } - importJsonUpdates(input) { - const parsed = typeof input === "string" ? JSON.parse(input) : input; - if (parsed.schema_version !== 1) { - throw new TypeError(`unsupported JSON schema version ${String(parsed.schema_version)}`); - } - const records = jsonSchemaToHistoryRecords(parsed); - return this.import(this.#encodeUpdates(records)); - } - import(bytes) { - this.#commit({}, true); - // Event inputs are only consumed by #emit, which returns immediately when - // nobody listens, so skip building them in that case. - const emitting = this.#hasEventSubscribers(); - const before = emitting ? this.#frontiersCodec() : []; - const beforeVersion = this.#historyVersion(); - const parsed = decodeDocument(bytes); - if (this.#deferredSnapshotHistory !== undefined) { - this.#materializeDeferredHistory(); - } - const imported = []; - let integration = { added: [], pending: [] }; - let beforeValues = new Map(); - let preparedDiffs = new Map(); - let deferredChanged; - let deferredStatus; - if (parsed.mode === EncodeMode.FastUpdates) { - for (const blockBytes of decodeFastUpdatesBody(parsed.body)) { - imported.push(...this.#readChangeBlock(blockBytes)); - } - this.#assertImportsNotOutdated(imported); - integration = this.#integrateHistory(imported); - const { added } = integration; - if (added.length > 0 && !this.#detached) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - if (recording !== undefined) { - this.#recordConcurrentTreeSubjects(recording, added, beforeVersion); - } - this.#applyRecords(added, recording); - this.#canonicalizeImportedMovableMoves(added, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - } - else { - const snapshot = decodeFastSnapshotBody(parsed.body); - const oplogEntries = decodeSstable(snapshot.oplog.slice()); - let encodedStartVersion; - let encodedStartFrontiers; - let encodedEndVersion; - let encodedEndFrontiers; - for (const entry of oplogEntries) { - if (bytesEqual(entry.key, START_VERSION_KEY)) { - encodedStartVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); - } - else if (bytesEqual(entry.key, START_FRONTIERS_KEY)) { - encodedStartFrontiers = decodePostcardFrontiers(entry.value); - } - else if (bytesEqual(entry.key, VERSION_KEY)) { - encodedEndVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); - } - else if (bytesEqual(entry.key, FRONTIERS_KEY)) { - encodedEndFrontiers = decodePostcardFrontiers(entry.value); - } - } - const initializeFromSnapshot = this.#history.size === 0 && - this.#pendingHistory.size === 0 && - this.#shallowRootStore === undefined; - const isShallowSnapshot = snapshot.shallowRootState.length > 0; - let rootStore; - let stagedShallowStartVersion; - let stagedShallowRootVersion; - let stagedShallowRootFrontiers; - let canonicalStartMetadata = !isShallowSnapshot && - encodedStartVersion === undefined && - encodedStartFrontiers === undefined; - if (initializeFromSnapshot && isShallowSnapshot) { - rootStore = decodeStateSnapshotStore(snapshot.shallowRootState.slice()); - if (rootStore.kind !== "sstable") { - throw new Error("shallow snapshot root state must be an SSTable"); - } - if (rootStore.frontiers !== undefined && - encodedStartFrontiers !== undefined && - !frontierSetsEqual(rootStore.frontiers.map(formatOpId), encodedStartFrontiers.map(formatOpId))) { - throw new Error("shallow snapshot start frontiers do not match root state"); - } - const startFrontiers = encodedStartFrontiers ?? rootStore.frontiers ?? []; - const startVersion = encodedStartVersion ?? new VersionVector(); - const rootVersion = startVersion.clone(); - for (const frontier of startFrontiers) { - rootVersion.set(frontier.peer, frontier.counter + 1); - } - stagedShallowStartVersion = startVersion; - stagedShallowRootVersion = rootVersion; - stagedShallowRootFrontiers = startFrontiers.map((id) => ({ ...id })); - canonicalStartMetadata = - encodedStartVersion !== undefined && encodedStartFrontiers !== undefined; - } - const stateStore = decodeStateSnapshotStore(snapshot.state); - const hydratedStore = rootStore === undefined - ? stateStore - : mergeStateSnapshotStores(rootStore, stateStore); - const preparedTextSnapshotIds = validateStateSnapshotTextIds(hydratedStore); - if (stagedShallowRootVersion !== undefined && - encodedEndVersion !== undefined && - !versionIncludes(encodedEndVersion, stagedShallowRootVersion)) { - throw new Error("shallow snapshot root version exceeds its end version"); - } - const installStagedShallowRoot = () => { - if (rootStore === undefined || - stagedShallowStartVersion === undefined || - stagedShallowRootVersion === undefined || - stagedShallowRootFrontiers === undefined) { - return; - } - this.#shallowStartVersion = stagedShallowStartVersion; - this.#shallowRootVersion = stagedShallowRootVersion; - this.#shallowRootFrontiers = stagedShallowRootFrontiers; - this.#shallowRootStore = rootStore; - }; - const canDeferHistory = initializeFromSnapshot && - !this.#detached && - canonicalStartMetadata && - encodedEndVersion !== undefined && - encodedEndFrontiers !== undefined && - stateStore.kind === "sstable" && - hydratedStore.kind === "sstable"; - if (canDeferHistory) { - const endVersion = encodedEndVersion; - const endFrontiers = encodedEndFrontiers; - this.#validateDeferredFrontierBlocks(oplogEntries, endVersion, endFrontiers); - installStagedShallowRoot(); - if (emitting) { - deferredChanged = new Set(hydratedStore.containers.map(({ id }) => formatContainerId(id))); - beforeValues = this.#captureContainerEventValues(deferredChanged); - } - this.#hydrateState(hydratedStore, preparedTextSnapshotIds); - this.#deferredSnapshotHistory = { - entries: oplogEntries, - endVersion, - frontiers: endFrontiers.map((id) => ({ ...id })), - operationCount: versionDistance(stagedShallowStartVersion ?? this.#shallowStartVersion, endVersion), - }; - for (const { peer } of endVersion._codecEntriesUnsorted()) { - this.#seenCommittedPeers.add(peer); - } - deferredStatus = importStatusBetweenVersions(beforeVersion, stagedShallowStartVersion ?? this.#shallowStartVersion, endVersion); - } - else { - for (const entry of oplogEntries) { - if (entry.key.length !== 12) - continue; - const expected = decodeChangeBlockKey(entry.key); - const records = this.#readChangeBlock(entry.value); - if (records[0] !== undefined && !idsEqual(records[0].change.id, expected)) { - throw new Error("snapshot change key does not match its block"); - } - imported.push(...records); - } - installStagedShallowRoot(); - if (!initializeFromSnapshot) - this.#assertImportsNotOutdated(imported); - integration = this.#integrateHistory(imported); - if (!this.#detached && initializeFromSnapshot) { - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureEventValues(integration.added); - } - if (rootStore !== undefined && stateStore.kind === "empty") { - this.#rebuildFromHistory(); - } - else { - this.#hydrateState(hydratedStore, preparedTextSnapshotIds); - } - } - else if (!this.#detached && integration.added.length > 0) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - if (recording !== undefined) { - this.#recordConcurrentTreeSubjects(recording, integration.added, beforeVersion); - } - this.#applyRecords(integration.added, recording); - this.#canonicalizeImportedMovableMoves(integration.added, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - } - } - this.#nextCounter = this.oplogVersion().get(this.#peer) ?? 0; - const changed = this.#detached || !emitting - ? new Set() - : (deferredChanged ?? - new Set(integration.added.flatMap(({ change }) => change.operations.map((op) => formatContainerId(op.container))))); - this.#emit("import", undefined, before, emitting ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); - return deferredStatus ?? importStatus(integration.added, integration.pending); - } - importBatch(blobs) { - if (blobs.length === 0) - return { success: new Map(), pending: null }; - if (blobs.length === 1) - return this.import(blobs[0]); - this.#commit({}, true); - this.#materializeDeferredHistory(); - const emitting = this.#hasEventSubscribers(); - const before = emitting ? this.#frontiersCodec() : []; - const ordered = blobs - .map((blob) => this.#decodeImportData(decodeDocument(blob))) - .sort((left, right) => Number(left.mode === EncodeMode.FastUpdates) - - Number(right.mode === EncodeMode.FastUpdates) || - right.records.length - left.records.length); - const initializeFromSnapshot = this.#history.size === 0 && - this.#pendingHistory.size === 0 && - this.#shallowRootStore === undefined; - const snapshotSeed = initializeFromSnapshot - ? ordered.find(({ snapshot }) => snapshot !== undefined) - : undefined; - let rootStore; - let stagedShallowRoot; - if (snapshotSeed?.snapshot !== undefined && - snapshotSeed.snapshot.shallowRootState.length > 0) { - rootStore = decodeStateSnapshotStore(snapshotSeed.snapshot.shallowRootState); - if (rootStore.kind !== "sstable") { - throw new Error("shallow snapshot root state must be an SSTable"); - } - const startFrontiers = snapshotSeed.startFrontiers ?? rootStore.frontiers ?? []; - const startVersion = snapshotSeed.startVersion ?? new VersionVector(); - const rootVersion = startVersion.clone(); - for (const frontier of startFrontiers) { - rootVersion.set(frontier.peer, frontier.counter + 1); - } - stagedShallowRoot = { - startVersion, - rootVersion, - frontiers: startFrontiers.map((id) => ({ ...id })), - }; - } - let snapshotStateStore; - let snapshotHydratedStore; - let preparedTextSnapshotIds; - if (snapshotSeed?.snapshot !== undefined) { - snapshotStateStore = decodeStateSnapshotStore(snapshotSeed.snapshot.state); - snapshotHydratedStore = - rootStore === undefined - ? snapshotStateStore - : mergeStateSnapshotStores(rootStore, snapshotStateStore); - preparedTextSnapshotIds = validateStateSnapshotTextIds(snapshotHydratedStore); - } - if (rootStore !== undefined && stagedShallowRoot !== undefined) { - this.#shallowStartVersion = stagedShallowRoot.startVersion; - this.#shallowRootVersion = stagedShallowRoot.rootVersion; - this.#shallowRootFrontiers = stagedShallowRoot.frontiers; - this.#shallowRootStore = rootStore; - } - for (const decoded of ordered) { - if (decoded !== snapshotSeed) - this.#assertImportsNotOutdated(decoded.records); - } - const integration = this.#integrateHistory(ordered.flatMap(({ records }) => records)); - let beforeValues = new Map(); - let preparedDiffs = new Map(); - if (integration.added.length > 0 && !this.#detached) { - if (snapshotSeed?.snapshot !== undefined) { - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureEventValues(integration.added); - } - if (rootStore !== undefined && snapshotStateStore.kind === "empty") { - this.#rebuildFromHistory(); - } - else { - this.#hydrateState(snapshotHydratedStore, preparedTextSnapshotIds); - const snapshotVersion = snapshotSeed.endVersion?.clone() ?? - historyVersionForRecords(snapshotSeed.records, snapshotSeed.startVersion); - const forwardRecords = this.#recordsInVersionRange(snapshotVersion, this.#historyVersion()); - this.#applyRecords(forwardRecords); - this.#canonicalizeImportedMovableMoves(forwardRecords); - } - } - else { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyRecords(integration.added, recording); - this.#canonicalizeImportedMovableMoves(integration.added, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - } - this.#nextCounter = this.oplogVersion().get(this.#peer) ?? 0; - const changed = this.#detached || !emitting - ? new Set() - : changedContainerIds(integration.added); - this.#emit("import", undefined, before, emitting ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); - return importStatus(integration.added, integration.pending); - } - importUpdateBatch(blobs) { - return this.importBatch(blobs); - } - toJSON() { - const output = {}; - for (const [name, container] of this.#roots) { - const value = container.toJSON(); - if (this.#hideEmptyRoots && isEmptyJson(value)) - continue; - output[name] = value; - } - return output; - } - toJsonWithReplacer(replacer) { - const processed = new Set(); - const run = (value) => { - if (Array.isArray(value)) { - return value.flatMap((item, index) => { - const next = visit(index, item); - return next === undefined ? [] : [next]; - }); - } - if (typeof value === "object" && value !== null && !(value instanceof Uint8Array)) { - return Object.fromEntries(Object.entries(value).flatMap(([key, item]) => { - const next = visit(key, item); - return next === undefined ? [] : [[key, next]]; - })); - } - return value; - }; - const visit = (key, value) => { - if (typeof value === "string" && - isContainerId(value) && - !processed.has(value)) { - const id = value; - processed.add(id); - const container = this.getContainerById(id); - if (container === undefined) - throw new RangeError(`container ${id} does not exist`); - const replaced = replacer(key, container); - if (replaced === container) - return run(container.getShallowValue()); - if (isContainer(replaced)) { - throw new TypeError("replacer cannot substitute a different container"); - } - return replaced === undefined ? undefined : run(replaced); - } - if (typeof value === "object" && value !== null && !(value instanceof Uint8Array)) { - return run(value); - } - const replaced = replacer(key, value); - if (isContainer(replaced)) { - throw new TypeError("replacer cannot introduce a container"); - } - return replaced; - }; - return run(this.getShallowValue()); - } - diff(from, to, forJson = true) { - this.#commit({}, true); - const fromVersion = this.#versionForExistingFrontiers(from); - const toVersion = this.#versionForExistingFrontiers(to); - this.#assertVersionNotBeforeShallowRoot(fromVersion); - this.#assertVersionNotBeforeShallowRoot(toVersion); - if (fromVersion.compare(toVersion) === 0) - return []; - const restoreVersion = this.version(); - const currentToFromForward = this.#recordsInVersionRange(restoreVersion, fromVersion); - const currentToFromRetreat = this.#recordsInVersionRange(fromVersion, restoreVersion); - const forwardRecords = this.#recordsInVersionRange(fromVersion, toVersion); - const retreatRecords = this.#recordsInVersionRange(toVersion, fromVersion); - const currentToFromMoveMode = movableMoveTransitionMode(currentToFromRetreat, currentToFromForward, this.#movableMovePeers); - const fromToToMoveMode = movableMoveTransitionMode(retreatRecords, forwardRecords, this.#movableMovePeers); - const useIncrementalTransition = this.#canTransitionRecords([...currentToFromRetreat, ...currentToFromForward], currentToFromMoveMode) && - this.#canTransitionRecords([...retreatRecords, ...forwardRecords], fromToToMoveMode); - let materializedVersion = restoreVersion; - let transitionFailed = false; - try { - if (materializedVersion.compare(fromVersion) !== 0) { - if (useIncrementalTransition) { - this.#applyVersionTransition(currentToFromRetreat, currentToFromForward, fromVersion, undefined, currentToFromMoveMode); - } - else { - this.#rebuildFromHistory(fromVersion); - } - materializedVersion = fromVersion; - } - const changed = changedContainerIds([...forwardRecords, ...retreatRecords]); - let before = new Map(); - let calculated = new Map(); - let mapKeysAtFrom = new Map(); - let mapKeysAtTo = new Map(); - if (useIncrementalTransition || retreatRecords.length === 0) { - const recording = { - beforeValues: new Map(), - eventStates: new Map(), - }; - if (useIncrementalTransition) { - this.#applyVersionTransition(retreatRecords, forwardRecords, toVersion, recording, fromToToMoveMode); - } - else { - this.#applyRecords(forwardRecords, recording); - } - before = recording.beforeValues; - calculated = this.#recordedEventDiffs(recording, true); - } - else { - before = this.#captureContainerEventValues(changed); - mapKeysAtFrom = this.#captureMapKeys(changed); - this.#rebuildFromHistory(toVersion); - mapKeysAtTo = this.#captureMapKeys(changed); - } - materializedVersion = toVersion; - return [...changed] - .flatMap((id) => { - const container = this.#containers.get(id); - return container === undefined ? [] : [container]; - }) - .sort((left, right) => containerDepth(left) - containerDepth(right)) - .flatMap((container) => { - const diff = calculated.get(container.id) ?? - containerDiff(container, before.get(container.id), { - from: mapKeysAtFrom.get(container.id), - to: mapKeysAtTo.get(container.id), - }); - if (isEmptyContainerDiff(diff)) - return []; - return [ - [container.id, forJson ? diffForJson(diff) : diff], - ]; - }); - } - catch (error) { - transitionFailed = useIncrementalTransition; - throw error; - } - finally { - if (transitionFailed) { - this.#rebuildFromHistory(restoreVersion); - } - else if (materializedVersion.compare(restoreVersion) !== 0) { - if (useIncrementalTransition) { - if (materializedVersion.compare(toVersion) === 0) { - this.#applyVersionTransition(forwardRecords, retreatRecords, fromVersion, undefined, fromToToMoveMode); - materializedVersion = fromVersion; - } - if (materializedVersion.compare(restoreVersion) !== 0) { - this.#applyVersionTransition(currentToFromForward, currentToFromRetreat, restoreVersion, undefined, currentToFromMoveMode); - } - } - else { - this.#rebuildFromHistory(restoreVersion); - } - } - } - } - applyDiff(diffBatch) { - if (!Array.isArray(diffBatch)) - throw new TypeError("diff batch must be an array"); - if (this.#detached && !this.#detachedEditing) { - throw new Error("cannot edit a detached document; call attach() first"); - } - const containerRemap = new Map(); - const treeRemap = new Map(); - for (const entry of diffBatch) { - if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string") { - throw new TypeError("each diff entry must be a [ContainerID, Diff] tuple"); - } - const sourceId = entry[0]; - const diff = entry[1]; - const container = this.#resolveDiffContainer(sourceId, containerRemap); - if (container === undefined) - continue; - this.#applyContainerDiff(container, diff, containerRemap, treeRemap); - } - } - revertTo(frontiers) { - if (this.#detached && !this.#detachedEditing) { - throw new Error("cannot edit a detached document; call attach() first"); - } - this.#commit({}, true); - const diff = this.diff(this.frontiers(), frontiers, false); - this.applyDiff(diff); - } - #resolveDiffContainer(sourceId, remap) { - const mapped = remap.get(sourceId); - if (mapped !== undefined) - return mapped; - const parsed = parseContainerId(sourceId); - if (parsed.kind === "root" && !isMergeableContainerId(parsed)) { - return this.#getOrCreateContainer(parsed); - } - const existing = this.#containers.get(formatContainerId(parsed)); - return existing !== undefined && !this._isContainerDeleted(existing) - ? existing - : undefined; - } - #applyContainerDiff(container, diff, containerRemap, treeRemap) { - if (diff.type === "map") { - if (!(container instanceof LoroMap)) - throw diffKindMismatch(container, diff.type); - for (const [key, value] of Object.entries(diff.updated)) { - if (value === undefined) { - container.delete(key); - continue; - } - const sourceChildId = diffContainerId(value); - if (sourceChildId === undefined) { - container.set(key, value); - } - else { - this.#applyMapChildDiff(container, key, sourceChildId, containerRemap); - } - } - return; - } - if (diff.type === "text") { - if (!(container instanceof LoroText)) - throw diffKindMismatch(container, diff.type); - container.applyDelta(diff.diff); - return; - } - if (diff.type === "counter") { - if (!(container instanceof LoroCounter)) - throw diffKindMismatch(container, diff.type); - container.increment(diff.increment); - return; - } - if (diff.type === "tree") { - if (!(container instanceof LoroTree)) - throw diffKindMismatch(container, diff.type); - this.#applyTreeDiff(container, diff.diff, containerRemap, treeRemap); - return; - } - if (!(container instanceof LoroList)) - throw diffKindMismatch(container, diff.type); - let position = 0; - for (const operation of diff.diff) { - if ("retain" in operation) { - position += operation.retain; - } - else if ("delete" in operation) { - container.delete(position, operation.delete); - } - else { - for (const value of operation.insert) { - const sourceChildId = diffContainerId(value); - if (sourceChildId === undefined) { - container.insert(position, value); - } - else { - const parsed = parseContainerId(sourceChildId); - if (parsed.kind === "root") { - throw new TypeError("a root container cannot be inserted as a child"); - } - const child = createContainer(codecTypeToPublic(parsed.containerType)); - const attached = container.insertContainer(position, child); - containerRemap.set(sourceChildId, attached); - } - position += 1; - } - } - } - } - #applyMapChildDiff(parent, key, sourceId, remap) { - const mapped = remap.get(sourceId); - const current = parent.get(key); - if (mapped !== undefined && current === mapped) - return; - const parsed = parseContainerId(sourceId); - const type = codecTypeToPublic(parsed.containerType); - if (isMergeableContainerId(parsed)) { - const child = ensureMergeableChild(parent, key, type); - remap.set(sourceId, child); - return; - } - if (parsed.kind === "root") { - throw new TypeError("a root container cannot be assigned as a child"); - } - if (isContainer(current) && current.id === sourceId) { - remap.set(sourceId, current); - return; - } - const child = createContainer(type); - remap.set(sourceId, parent.setContainer(key, child)); - } - #applyTreeDiff(tree, diff, containerRemap, treeRemap) { - const resolveNode = (source) => treeRemap.get(source) ?? (tree.has(source) ? source : undefined); - for (const item of diff) { - if (item.action !== "delete") - continue; - const target = resolveNode(item.target); - if (target !== undefined) - tree.delete(target); - } - const pending = diff.filter((item) => item.action !== "delete"); - while (pending.length > 0) { - let progressed = false; - for (let index = 0; index < pending.length;) { - const item = pending[index]; - const parent = item.parent === undefined ? undefined : resolveNode(item.parent); - if (item.parent !== undefined && parent === undefined) { - index += 1; - continue; - } - if (item.action === "create") { - const node = tree.createNode(parent, item.index); - treeRemap.set(item.target, node.id); - const sourceNode = parseTreeId(item.target); - const sourceMetaId = formatContainerId({ - kind: "normal", - ...sourceNode, - containerType: CodecContainerType.Map, - }); - containerRemap.set(sourceMetaId, node.data); - } - else { - const target = resolveNode(item.target); - if (target !== undefined) - tree.move(target, parent, item.index); - } - pending.splice(index, 1); - progressed = true; - } - if (!progressed) { - throw new RangeError("tree diff refers to a parent that does not exist"); - } - } - } - getDeepValueWithId() { - return Object.fromEntries([...this.#roots].map(([name, container]) => [ - name, - containerDeepValueWithId(container), - ])); - } - getShallowValue() { - return Object.fromEntries([...this.#roots].map(([name, container]) => [name, container.id])); - } - getDeepValueWithID() { - return this.getDeepValueWithId(); - } - version() { - const version = this.#checkoutVersion?.clone() ?? this.#historyVersion(); - if (this.#pending !== undefined) { - const end = this.#pending.id.counter + this.#pending.operationLength; - if (end > (version.get(this.#peer) ?? 0)) - version.set(this.#peer, end); - } - return version; - } - oplogVersion() { - return this.#historyVersion(); - } - #historyVersion() { - if (this.#deferredSnapshotHistory !== undefined) { - return this.#deferredSnapshotHistory.endVersion.clone(); - } - const version = this.#shallowStartVersion.clone(); - for (const [peer, end] of this.#historyEndByPeer) { - if (end > (version.get(peer) ?? 0)) - version.set(peer, end); - } - return version; - } - frontiers() { - return this.#frontiersCodec().map(formatOpId); - } - oplogFrontiers() { - if (this.#deferredSnapshotHistory !== undefined) { - return [...this.#deferredSnapshotHistory.frontiers] - .sort(compareIds) - .map(formatOpId); - } - return [...this.#historyFrontiers.values()].sort(compareIds).map(formatOpId); - } - frontiersToVV(frontiers) { - const version = new VersionVector(); - for (const [peer, counter] of this.#causalVersionAt(frontiers.map(parseOpId))) { - version.set(peer, counter); - } - return version; - } - #versionForExistingFrontiers(frontiers) { - for (const frontier of frontiers) { - if (this.#recordContaining(parseOpId(frontier)) === undefined) { - throw new RangeError(`frontiers include unknown id ${frontier.counter}@${frontier.peer}`); - } - } - return this.frontiersToVV(frontiers); - } - vvToFrontiers(version) { - return this.#frontiersForVersion(version).map(formatOpId); - } - cmpWithFrontiers(frontiers) { - const current = this.oplogFrontiers(); - if (frontierSetsEqual(current, frontiers)) - return 0; - const version = this.#historyVersion(); - return frontiers.every((frontier) => { - const id = parseOpId(frontier); - return id.counter < (version.get(id.peer) ?? 0); - }) - ? 1 - : -1; - } - cmpFrontiers(left, right) { - return this.#versionForExistingFrontiers(left).compare(this.#versionForExistingFrontiers(right)); - } - changeCount() { - this.#materializeDeferredHistory(); - return this.#history.size; - } - debugHistory() { - this.#sortedHistory(); - } - opCount() { - return this.#deferredSnapshotHistory?.operationCount ?? this.#historyOperationCount; - } - getAllChanges() { - const changes = new Map(); - for (const { change } of this.#sortedHistory()) { - const peer = peerIdToString(change.id.peer); - const peerChanges = changes.get(peer); - if (peerChanges === undefined) - changes.set(peer, [publicChange(change)]); - else - peerChanges.push(publicChange(change)); - } - return changes; - } - getChangeAt(id) { - const target = parseOpId(id); - const record = this.#recordContaining(target); - if (record === undefined) { - throw new RangeError(`change ${target.counter}@${target.peer} is unknown`); - } - return publicChange(record.change); - } - getChangeAtLamport(peer, lamport) { - this.#materializeDeferredHistory(); - const parsedPeer = parsePeerId(peer); - const records = this.#historyByPeer.get(parsedPeer) ?? []; - let low = 0; - let high = records.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (records[middle].change.lamport <= lamport) - low = middle + 1; - else - high = middle; - } - const record = low > 0 ? records[low - 1] : undefined; - return record === undefined ? undefined : publicChange(record.change); - } - getOpsInChange(id) { - this.#commit({}, true); - const record = this.#recordContaining(parseOpId(id)); - if (record === undefined) { - throw new RangeError(`change ${id.counter}@${id.peer} is unknown`); - } - return historyRecordToJsonChange(record, undefined, false).ops; - } - getUncommittedOpsAsJson() { - const pending = this.#pending; - if (pending === undefined || pending.operations.length === 0) - return undefined; - const timestamp = this.#nextCommitOptions.timestamp ?? - (this.#recordTimestamp ? Date.now() / 1000 : 0); - const record = { - change: { - id: pending.id, - dependencies: pending.dependencies, - lamport: pending.lamport, - timestamp: numberToI64(timestamp), - message: this.#nextCommitOptions.message, - operations: pending.operations, - }, - keys: pending.keys, - keyIndices: undefined, - }; - return historyRecordsToJsonSchema([record], this.#historyVersion(), false); - } - travelChangeAncestors(ids, callback) { - this.#commit({}, true); - const visited = new Set(); - let stopped = false; - const visit = (id) => { - if (stopped) - return; - const record = this.#recordContaining(id); - if (record === undefined) { - if (id.counter < (this.#shallowStartVersion.get(id.peer) ?? 0)) - return; - throw new RangeError(`change ${id.counter}@${id.peer} is unknown`); - } - const key = changeKey(record.change.id); - if (visited.has(key)) - return; - for (const dependency of record.change.dependencies) - visit(dependency); - if (stopped || visited.has(key)) - return; - visited.add(key); - if (callback(publicChange(record.change)) === false) - stopped = true; - }; - for (const id of ids.map(parseOpId)) - visit(id); - } - findIdSpansBetween(from, to) { - const fromVersion = this.#versionForExistingFrontiers(from); - const toVersion = this.#versionForExistingFrontiers(to); - this.#assertVersionNotBeforeShallowRoot(fromVersion); - this.#assertVersionNotBeforeShallowRoot(toVersion); - const retreat = []; - const forward = []; - const peers = new Set([ - ...fromVersion._codecEntriesUnsorted().map(({ peer }) => peer), - ...toVersion._codecEntriesUnsorted().map(({ peer }) => peer), - ]); - for (const peer of [...peers].sort((left, right) => left < right ? -1 : left > right ? 1 : 0)) { - const fromCounter = fromVersion.get(peer) ?? 0; - const toCounter = toVersion.get(peer) ?? 0; - if (fromCounter < toCounter) { - forward.push({ - peer: peerIdToString(peer), - counter: fromCounter, - length: toCounter - fromCounter, - }); - } - else if (fromCounter > toCounter) { - retreat.push({ - peer: peerIdToString(peer), - counter: toCounter, - length: fromCounter - toCounter, - }); - } - } - return { retreat, forward }; - } - getChangedContainersIn(id, len) { - this.#commit({}, true); - this.#materializeDeferredHistory(); - if (!Number.isSafeInteger(len) || len < 0) { - throw new RangeError(`change range length is out of range: ${len}`); - } - const start = parseOpId(id); - const end = start.counter + len; - const containers = new Set(); - const records = this.#historyByPeer.get(start.peer) ?? []; - let recordIndex = Math.max(0, lowerBoundHistory(records, start.counter + 1) - 1); - for (; recordIndex < records.length; recordIndex += 1) { - const { change } = records[recordIndex]; - if (change.id.counter >= end) - break; - let operationIndex = lowerBoundOperation(change.operations, start.counter + 1) - 1; - operationIndex = Math.max(0, operationIndex); - for (; operationIndex < change.operations.length; operationIndex += 1) { - const operation = change.operations[operationIndex]; - if (operation.counter >= end) - break; - if (operation.counter < end && - operation.counter + operation.length > start.counter) { - containers.add(formatContainerId(operation.container)); - } - } - } - return [...containers]; - } - #recordContaining(id) { - this.#materializeDeferredHistory(); - const records = this.#historyByPeer.get(id.peer); - if (records === undefined) - return undefined; - const index = lowerBoundHistory(records, id.counter + 1) - 1; - const record = index >= 0 ? records[index] : undefined; - return record !== undefined && - id.counter < record.change.id.counter + changeLength(record.change) - ? record - : undefined; - } - #greatestTimestampAt(frontiers) { - const version = this.#causalVersionAt(frontiers); - let greatest = 0n; - for (const [peer, counter] of version) { - if (counter === 0) - continue; - const record = this.#recordContaining({ peer, counter: counter - 1 }); - if (record !== undefined) - greatest = maxBigInt(greatest, record.change.timestamp); - } - return greatest; - } - #mergeablePreviousRecord(change, interval) { - if (change.id.counter === 0) - return undefined; - const previous = this.#recordContaining({ - peer: change.id.peer, - counter: change.id.counter - 1, - }); - return previous !== undefined && canMergeChanges(previous.change, change, interval) - ? previous - : undefined; - } - fork() { - return this.forkAt(this.frontiers()); - } - forkAt(frontiers) { - this.#commit({}, true); - const version = this.#versionForExistingFrontiers(frontiers); - this.#assertVersionNotBeforeShallowRoot(version); - const fork = new LoroDoc(); - fork.#recordTimestamp = this.#recordTimestamp; - fork.#changeMergeInterval = this.#changeMergeInterval; - fork.#detachedEditing = this.#detachedEditing; - fork.#hideEmptyRoots = this.#hideEmptyRoots; - fork.#textStyles = new Map(this.#textStyles); - fork.#defaultTextStyle = this.#defaultTextStyle; - fork.#shallowStartVersion = this.#shallowStartVersion.clone(); - fork.#shallowRootVersion = this.#shallowRootVersion.clone(); - fork.#shallowRootFrontiers = this.#shallowRootFrontiers.map((id) => ({ ...id })); - fork.#shallowRootStore = this.#shallowRootStore; - fork.#integrateHistory(this.#recordsAtVersion(version), undefined); - fork.#rebuildFromHistory(); - return fork; - } - attach() { - this.checkoutToLatest(); - } - checkout(frontiers) { - this.#commit({}, true); - for (const frontier of frontiers) { - if (this.#recordContaining(parseOpId(frontier)) === undefined) { - throw new RangeError(`frontier ${frontier.counter}@${frontier.peer} is unknown`); - } - } - const before = this.#hasEventSubscribers() ? this.#frontiersCodec() : []; - const currentVersion = this.version(); - const targetVersion = this.frontiersToVV(frontiers); - this.#assertVersionNotBeforeShallowRoot(targetVersion); - const forwardRecords = this.#recordsInVersionRange(currentVersion, targetVersion); - const retreatRecords = this.#recordsInVersionRange(targetVersion, currentVersion); - const changed = this.#hasEventSubscribers() - ? changedContainerIds([...forwardRecords, ...retreatRecords]) - : new Set(); - let beforeValues = new Map(); - let preparedDiffs = new Map(); - this.#checkoutVersion = targetVersion; - this.#detached = true; - const changedRecords = [...forwardRecords, ...retreatRecords]; - const movableMoveMode = movableMoveTransitionMode(retreatRecords, forwardRecords, this.#movableMovePeers); - if (this.#canTransitionRecords(changedRecords, movableMoveMode)) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyVersionTransition(retreatRecords, forwardRecords, targetVersion, recording, movableMoveMode); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - else if (retreatRecords.length === 0 && - !hasMaterializedSequenceInsertions(forwardRecords, this.#containers)) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyRecords(forwardRecords, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - else { - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureContainerEventValues(changed); - } - this.#rebuildFromHistory(targetVersion); - } - this.#emit("checkout", undefined, before, changed.size > 0 ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); - if (this.#detachedEditing) - this.#renewPeerId(); - } - detach() { - this.#commit({}, true); - this.#checkoutVersion = this.version(); - this.#detached = true; - } - isDetached() { - return this.#detached; - } - checkoutToLatest() { - this.#commit({}, true); - const wasDetached = this.#detached; - // Without subscribers (or when already attached) no event is emitted, so - // skip building the frontier array and the changed-container set. - const emitting = wasDetached && this.#hasEventSubscribers(); - const before = emitting ? this.#frontiersCodec() : []; - const currentVersion = this.version(); - const latestVersion = this.#historyVersion(); - const forwardRecords = this.#recordsInVersionRange(currentVersion, latestVersion); - const retreatRecords = this.#recordsInVersionRange(latestVersion, currentVersion); - const changed = emitting - ? changedContainerIds([...forwardRecords, ...retreatRecords]) - : new Set(); - let beforeValues = new Map(); - let preparedDiffs = new Map(); - this.#checkoutVersion = undefined; - this.#detached = false; - if (wasDetached) { - const changedRecords = [...forwardRecords, ...retreatRecords]; - const movableMoveMode = movableMoveTransitionMode(retreatRecords, forwardRecords, this.#movableMovePeers); - if (this.#canTransitionRecords(changedRecords, movableMoveMode)) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyVersionTransition(retreatRecords, forwardRecords, latestVersion, recording, movableMoveMode); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - else if (retreatRecords.length === 0 && - !hasMaterializedSequenceInsertions(forwardRecords, this.#containers)) { - const recording = this.#hasEventSubscribers() - ? { beforeValues: new Map(), eventStates: new Map() } - : undefined; - this.#applyRecords(forwardRecords, recording); - if (recording !== undefined) { - beforeValues = recording.beforeValues; - preparedDiffs = this.#recordedEventDiffs(recording); - } - } - else { - if (this.#hasEventSubscribers()) { - beforeValues = this.#captureContainerEventValues(changed); - } - this.#rebuildFromHistory(); - } - this.#emit("checkout", undefined, before, changed.size > 0 ? this.#frontiersCodec() : [], changed, beforeValues, preparedDiffs); - if (this.#detachedEditing) - this.#renewPeerId(); - } - } - #renewPeerId() { - let peer = generatePeerId(); - while (peer === this.#peer || peer === 0xffffffffffffffffn) { - peer = generatePeerId(); - } - this.#peer = peer; - this.#nextCounter = this.oplogVersion().get(peer) ?? 0; - } - isShallow() { - return this.#shallowRootStore !== undefined; - } - shallowSinceVV() { - return this.#shallowStartVersion.clone(); - } - shallowSinceFrontiers() { - return this.#shallowRootFrontiers.map(formatOpId); - } - subscribe(listener) { - this.#subscribers.add(listener); - return () => this.#subscribers.delete(listener); - } - subscribeLocalUpdates(listener) { - this.#localUpdateSubscribers.add(listener); - return () => this.#localUpdateSubscribers.delete(listener); - } - subscribeFirstCommitFromPeer(listener) { - this.#firstCommitSubscribers.add(listener); - return () => this.#firstCommitSubscribers.delete(listener); - } - subscribePreCommit(listener) { - this.#preCommitSubscribers.add(listener); - return () => this.#preCommitSubscribers.delete(listener); - } - subscribeJsonpath(path, callback) { - const matches = compileJsonPathEventMatcher(path); - return this.subscribe((event) => { - if (event.events.some(matches)) - callback(); - }); - } - JSONPath(path) { - if (path === "$") - return [Object.fromEntries(this.#roots)]; - return evaluateJsonPath(this.#roots, path); - } - getPathToContainer(id) { - const container = this.getContainerById(id); - if (container === undefined || this._isContainerDeleted(container)) - return undefined; - return containerPath(container); - } - getByPath(path) { - const parts = path.split("/").filter(Boolean); - if (parts.length === 0) - return undefined; - let value = this.#roots.get(parts[0]); - for (const part of parts.slice(1)) { - if (value instanceof LoroMap) { - value = value.get(part); - } - else if (value instanceof LoroList) { - value = value.get(parsePathIndex(part)); - } - else if (value instanceof LoroTree) { - value = treeNodeAtPath(value, undefined, part); - } - else if (value instanceof LoroTreeNode) { - const index = parseOptionalPathIndex(part); - value = index === undefined ? value.data.get(part) : value._childAt(index); - } - else if (Array.isArray(value)) { - value = value[parsePathIndex(part)]; - } - else if (typeof value === "object" && value !== null) { - value = value[part]; - } - else { - return undefined; - } - if (value === undefined) - return undefined; - } - return value instanceof LoroTreeNode ? value.data : value; - } - getCursorPos(cursor) { - const container = this.getContainerById(cursor.containerId()); - if (container === undefined || this._isContainerDeleted(container)) - return undefined; - const id = cursor._idValue(); - if (!(container instanceof LoroList || container instanceof LoroText)) - return undefined; - const publicLength = container.length; - if (id === undefined) { - return { - offset: Math.min(cursor._originPositionValue(), publicLength), - side: cursor.side(), - }; - } - const target = container._sequence.findById(id); - if (target === undefined) - return undefined; - const publicOffset = (element) => container instanceof LoroText - ? (container._sequence.visibleMetricOffsetOf(element, "utf16") ?? - 0) - : (container._sequence.visibleIndexOf(element) ?? 0); - if (!target.deleted) { - const offset = publicOffset(target); - return { - offset: offset + - (cursor.side() === 1 - ? container instanceof LoroText - ? target.value.length - : 1 - : 0), - side: cursor.side(), - }; - } - const next = container._sequence.nextVisible(target); - if (next !== undefined) { - const offset = publicOffset(next); - return { - offset, - side: -1, - update: new Cursor(container.id, next.id, -1, offset), - }; - } - const previous = container._sequence.previousVisible(target); - if (previous !== undefined) { - return { - offset: publicLength, - side: 1, - update: new Cursor(container.id, previous.id, 1, publicLength), - }; - } - return { - offset: 0, - side: 0, - update: new Cursor(container.id, undefined, 0, 0), - }; - } - _subscribeContainer(container, listener) { - let listeners = this.#containerSubscribers.get(container.id); - if (listeners === undefined) { - listeners = new Set(); - this.#containerSubscribers.set(container.id, listeners); - } - listeners.add(listener); - return () => { - listeners.delete(listener); - if (listeners.size === 0 && - this.#containerSubscribers.get(container.id) === listeners) { - this.#containerSubscribers.delete(container.id); - } - }; - } - _isContainerDeleted(container) { - if (container._codecId?.kind === "root" && - !isMergeableContainerId(container._codecId)) - return !this.#roots.has(container._codecId.name); - const parent = container.parent(); - if (container._codecId !== undefined && - isMergeableContainerId(container._codecId) && - parent === undefined) - return true; - const binding = container._parentLink?.binding ?? - (parent === undefined ? undefined : recoverParentBinding(container, parent)); - if (binding?.kind === "map" && parent instanceof LoroMap) { - const record = parent._entries.get(binding.key); - return record === undefined || record.deleted || record.value !== container; - } - if (binding?.kind === "sequence" && parent instanceof LoroList) { - return binding.element.deleted || binding.element.value !== container; - } - if (binding?.kind === "tree" && parent instanceof LoroTree) { - return parent._isRecordDeleted(binding.record) || binding.record.data !== container; - } - return parent !== undefined; - } - _undoIdSpan(peer, range) { - this.#commit({}, true); - this.#materializeDeferredHistory(); - const parsedPeer = parsePeerId(peer); - const records = this.#historyByPeer.get(parsedPeer) ?? []; - let recordIndex = Math.min(records.length - 1, lowerBoundHistory(records, range.end) - 1); - for (; recordIndex >= 0; recordIndex -= 1) { - const record = records[recordIndex]; - const changeEnd = record.change.id.counter + changeLength(record.change); - if (changeEnd <= range.start) - break; - let operationIndex = Math.min(record.change.operations.length - 1, lowerBoundOperation(record.change.operations, range.end) - 1); - for (; operationIndex >= 0; operationIndex -= 1) { - const operation = record.change.operations[operationIndex]; - if (operation.counter + operation.length <= range.start) - break; - this.#undoOperation(record, operation, parsedPeer, range); - } - } - } - _transformUndoCursors(cursors) { - return cursors.map((cursor) => { - const container = this.getContainerById(cursor.containerId()); - if (!(container instanceof LoroText || container instanceof LoroList)) { - return cursor; - } - const position = Math.min(cursor._originPositionValue(), container.length); - return container.getCursor(position, cursor.side()) ?? cursor; - }); - } - #undoOperation(record, operation, peer, range) { - const container = this.#containers.get(formatContainerId(operation.container)); - if (container === undefined) - return; - const content = operation.content; - const overlapStart = Math.max(operation.counter, range.start); - const overlapEnd = Math.min(operation.counter + operation.length, range.end); - if (overlapStart >= overlapEnd) - return; - switch (content.type) { - case "text-insert": - if (container instanceof LoroText) { - this.#deleteInsertedElements(container, peer, overlapStart, overlapEnd); - } - return; - case "list-insert": - case "movable-list-insert": - if (container instanceof LoroList) { - this.#deleteInsertedElements(container, peer, overlapStart, overlapEnd); - } - return; - case "text-delete": - if (container instanceof LoroText) { - this.#restoreDeletedElements(container, peer, range, overlapStart, overlapEnd); - } - return; - case "list-delete": - case "movable-list-delete": - if (container instanceof LoroList) { - this.#restoreDeletedElements(container, peer, range, overlapStart, overlapEnd); - } - return; - case "map-insert": - case "map-delete": - if (container instanceof LoroMap) { - this.#undoMapOperation(container, record, operation); - } - return; - case "tree-create": - case "tree-move": - case "tree-delete": - if (container instanceof LoroTree) { - this.#undoTreeOperation(container, record, operation); - } - return; - case "future": - if (container instanceof LoroCounter && - (content.value.type === "double" || content.value.type === "i64")) { - container.increment(-(content.value.type === "double" - ? content.value.value - : Number(content.value.value))); - } - else if (container instanceof LoroCounter && - content.value.type === "delta-int") { - container.increment(-content.value.value); - } - return; - case "text-mark": - case "text-mark-end": - case "movable-list-move": - case "movable-list-set": - return; - } - } - #deleteInsertedElements(container, peer, start, end) { - const positions = []; - for (let counter = start; counter < end; counter += 1) { - const element = container._sequence.findById({ peer, counter }); - if (element === undefined || element.deleted) - continue; - const index = container._sequence.visibleIndexOf(element); - if (index !== undefined) - positions.push(index); - } - positions.sort((left, right) => left - right); - const ranges = []; - for (const position of positions) { - const previous = ranges.at(-1); - if (previous !== undefined && previous.start + previous.length === position) { - previous.length += 1; - } - else { - ranges.push({ start: position, length: 1 }); - } - } - for (const selected of ranges.reverse()) { - if (container instanceof LoroText) { - this._textDelete(container, selected.start, selected.length); - } - else { - this._sequenceDelete(container, selected.start, selected.length); - } - } - } - #restoreDeletedElements(container, peer, range, overlapStart, overlapEnd) { - const belongsToUndo = (id) => id.peer === peer && id.counter >= range.start && id.counter < range.end; - const belongsToOperation = (id) => id.peer === peer && id.counter >= overlapStart && id.counter < overlapEnd; - const targets = container._sequence - .elementsDeletedBy(peer, overlapStart, overlapEnd) - .filter((element) => !belongsToUndo(element.id) && - container._sequence.someDeletion(element, belongsToOperation) && - container._sequence.everyDeletion(element, belongsToUndo)); - const groups = []; - for (const target of targets) { - const previous = groups.at(-1)?.at(-1); - if (previous !== undefined && - container._sequence.physicalIndexOf(previous) + 1 === - container._sequence.physicalIndexOf(target)) { - groups.at(-1).push(target); - } - else { - groups.push([target]); - } - } - for (const group of groups) { - const position = container._sequence.visibleIndexOf(group[0]); - if (container instanceof LoroText) { - const text = group - .map((element) => { - if (typeof element.value !== "string") { - throw new TypeError("text element value must be a string"); - } - return element.value; - }) - .join(""); - this._textInsert(container, position, text); - } - else { - let offset = 0; - for (const element of group) { - if (isContainer(element.value)) { - const child = createContainer(element.value.kind()); - restoreBlueprint(child, captureBlueprint(element.value)); - this._listInsertContainer(container, position + offset, child); - } - else { - this._listInsert(container, position + offset, cloneRuntimeValue(element.value)); - } - offset += 1; - } - } - } - } - #undoMapOperation(map, record, operation) { - const content = operation.content; - if (content.type !== "map-insert" && content.type !== "map-delete") - return; - const writer = operationWriter(record.change, operation); - const current = map._entries.get(content.key); - if (current === undefined || compareWriter(current.writer, writer) !== 0) - return; - const previous = this.#previousMapOperation(operation.container, content.key, writer); - if (previous === undefined || previous.operation.content.type === "map-delete") { - map.delete(content.key); - return; - } - const previousContent = previous.operation.content; - if (previousContent.type !== "map-insert") - return; - const previousId = { - peer: previous.record.change.id.peer, - counter: previous.operation.counter, - }; - const rawValue = this.#decodeRuntimeValue(previousContent.value, previous.record.keys, previousId, map); - const value = this.#materializeMapValue(map, content.key, rawValue); - if (isContainer(value)) { - const child = createContainer(value.kind()); - restoreBlueprint(child, captureBlueprint(value)); - map.setContainer(content.key, child); - } - else { - map.set(content.key, cloneRuntimeValue(value)); - } - } - #previousMapOperation(containerId, key, before) { - const operations = this.#mapOperationHistory - .get(formatContainerId(containerId)) - ?.get(key); - if (operations === undefined) - return undefined; - const previous = operations.byWriter.at(operations.byWriter._lowerBoundBy((operation) => compareWriter(operation.writer, before)) - 1); - return previous === undefined - ? undefined - : { record: previous.record, operation: previous.operation }; - } - #undoTreeOperation(tree, record, operation) { - const content = operation.content; - if (content.type !== "tree-create" && - content.type !== "tree-move" && - content.type !== "tree-delete") { - return; - } - const writer = operationWriter(record.change, operation); - const node = tree._nodes.get(formatTreeId(content.subject)); - if (node === undefined || compareWriter(node.writer, writer) !== 0) - return; - const previous = this.#previousTreeOperation(operation.container, content.subject, writer); - if (previous === undefined || previous.operation.content.type === "tree-delete") { - this.#appendAndApply(tree, { type: "tree-delete", subject: content.subject }, 1); - return; - } - const previousContent = previous.operation.content; - if (previousContent.type !== "tree-create" && previousContent.type !== "tree-move") { - return; - } - this.#appendAndApply(tree, { - type: "tree-move", - subject: content.subject, - parent: previousContent.parent, - position: previousContent.position.slice(), - }, 1); - } - #previousTreeOperation(containerId, subject, before) { - const operations = this.#treeOperationHistory - .get(formatContainerId(containerId)) - ?.get(idKey(subject)); - if (operations === undefined) - return undefined; - const previous = operations.byWriter.at(operations.byWriter._lowerBoundBy((operation) => compareWriter(operation.writer, before)) - 1); - return previous === undefined - ? undefined - : { record: previous.record, operation: previous.operation }; - } - _mapSet(container, key, value) { - const current = container._entries.get(key); - if (current !== undefined && - !current.deleted && - eventValuesEqual(current.value, value)) { - return; - } - const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); - this.#appendAndApply(container, { type: "map-insert", key, value: encoded }, 1); - } - _mapDelete(container, key) { - const current = container._entries.get(key); - if (current === undefined || current.deleted) - return; - this.#appendAndApply(container, { type: "map-delete", key }, 1); - } - _mapSetContainer(container, key, child) { - return this.#attachChild(container, child, (rawType) => ({ - type: "map-insert", - key, - value: { type: "container-type", value: rawType }, - })); - } - _mapEnsureMergeable(container, key, type) { - const parentId = container._codecId; - if (parentId === undefined || container._doc !== this) { - throw new Error("cannot ensure a mergeable child on a detached map"); - } - const codecType = publicTypeToCodec(type); - const childId = newMergeableContainerId(parentId, key, codecType); - const marker = mergeableMarker(parentId, key, codecType); - const existing = container._entries.get(key); - if (existing !== undefined && !existing.deleted && existing.rawValue !== null) { - const existingType = parseMergeableMarker(parentId, key, existing.rawValue); - if (existingType === undefined) { - throw new TypeError(`cannot create a mergeable ${type} at key ${JSON.stringify(key)}: ` + - "the key already holds a non-mergeable value"); - } - if (existing.rawValue instanceof Uint8Array && - bytesEqual(existing.rawValue, marker)) { - return this.#getOrCreateContainer(childId, container); - } - } - const child = this.#getOrCreateContainer(childId, container); - this.#appendAndApply(container, { type: "map-insert", key, value: { type: "binary", value: marker } }, 1); - return child; - } - _listInsert(container, position, value) { - const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); - const type = container instanceof LoroMovableList ? "movable-list-insert" : "list-insert"; - this.#appendAndApply(container, { type, position, values: [encoded] }, 1); - } - _listInsertContainer(container, position, child) { - return this.#attachChild(container, child, (rawType) => ({ - type: container instanceof LoroMovableList ? "movable-list-insert" : "list-insert", - position, - values: [{ type: "container-type", value: rawType }], - })); - } - _sequenceDelete(container, position, length) { - this.#deleteSequenceRuns(container, position, length, container instanceof LoroMovableList ? "movable-list-delete" : "list-delete"); - } - _textInsert(container, position, text) { - this.#appendAndApply(container, { type: "text-insert", position, value: text }, unicodeScalarLength(text)); - } - _textDelete(container, position, length) { - this.#deleteTextRuns(container, position, length); - } - _textMark(container, start, end, key, value) { - const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); - const info = textStyleInfoByte(this.#textStyleExpand(key), value === null); - this.#appendAndApply(container, { type: "text-mark", start, end, key, value: encoded, info }, 1); - this.#appendAndApply(container, { type: "text-mark-end" }, 1); - } - _movableMove(container, from, to) { - const element = container._visibleElementAt(from); - this.#appendAndApply(container, { - type: "movable-list-move", - from, - to, - elementId: { peer: element.id.peer, lamport: element.lamport }, - }, 1); - } - _movableSet(container, position, value) { - const element = container._visibleElementAt(position); - if (eventValuesEqual(element.value, value)) - return; - const encoded = this.#encodeRuntimeValue(value, this.#ensurePending()); - this.#appendAndApply(container, { - type: "movable-list-set", - elementId: { peer: element.id.peer, lamport: element.lamport }, - value: encoded, - }, 1); - } - _movableSetContainer(container, position, child) { - const element = container._visibleElementAt(position); - return this.#attachChild(container, child, (rawType) => ({ - type: "movable-list-set", - elementId: { peer: element.id.peer, lamport: element.lamport }, - value: { type: "container-type", value: rawType }, - })); - } - _counterIncrement(container, value) { - if (value === 0) - return; - this.#appendAndApply(container, { type: "future", property: 0, value: { type: "double", value } }, 1); - } - #textStyleExpand(key) { - const baseKey = key.split(":", 1)[0]; - const configured = this.#textStyles.get(baseKey) ?? this.#defaultTextStyle; - if (configured === undefined) { - throw new RangeError(`text style ${JSON.stringify(baseKey)} is not configured`); - } - return configured; - } - _treeCreate(tree, parent, index) { - const pending = this.#ensurePending(); - const subject = { peer: this.#peer, counter: this.#nextOperationCounter(pending) }; - const parentId = parent === undefined ? undefined : parseTreeId(parent); - if (parent !== undefined && !tree.has(parent)) { - throw new RangeError(`tree parent ${parent} does not exist`); - } - const position = tree._positionFor(parentId, index); - this.#appendAndApply(tree, { type: "tree-create", subject, parent: parentId, position }, 1); - return new LoroTreeNode(tree, subject); - } - _treeMove(tree, target, parent, index) { - if (!tree.has(target)) - throw new RangeError(`tree node ${target} does not exist`); - if (parent !== undefined && !tree.has(parent)) { - throw new RangeError(`tree parent ${parent} does not exist`); - } - const subject = parseTreeId(target); - const parentId = parent === undefined ? undefined : parseTreeId(parent); - let ancestor = parentId; - while (ancestor !== undefined) { - if (idsEqual(ancestor, subject)) { - throw new RangeError("cannot move a tree node below itself or its descendant"); - } - ancestor = tree._nodes.get(formatTreeId(ancestor))?.parent; - } - const position = tree._positionFor(parentId, index, subject); - this.#appendAndApply(tree, { type: "tree-move", subject, parent: parentId, position }, 1); - } - _treeDelete(tree, target) { - this.#appendAndApply(tree, { type: "tree-delete", subject: parseTreeId(target) }, 1); - } - #getContainer(nameOrId, expectedType) { - const byId = isContainerId(nameOrId); - const id = byId - ? parseContainerId(nameOrId) - : { kind: "root", name: nameOrId, containerType: publicTypeToCodec(expectedType) }; - if (codecTypeToPublic(id.containerType) !== expectedType) { - throw new TypeError(`container ${nameOrId} is not a ${expectedType}`); - } - if (byId && - (id.kind === "normal" || isMergeableContainerId(id)) && - !this.#containers.has(formatContainerId(id))) { - throw new RangeError(`container ${nameOrId} does not exist in this document`); - } - const container = this.#getOrCreateContainer(id); - if (id.kind === "root" && !isMergeableContainerId(id)) { - this.#roots.set(id.name, container); - } - return container; - } - #getOrCreateContainer(id, parent) { - const key = this.#containerKey(id); - const existing = this.#containers.get(key); - if (existing !== undefined) { - if (parent !== undefined) - existing._parentLink = { container: parent }; - return existing; - } - const container = createContainer(codecTypeToPublic(id.containerType)); - container._attach(this, id, parent); - this.#containers.set(key, container); - if (id.kind === "root" && !isMergeableContainerId(id)) { - this.#roots.set(id.name, container); - } - return container; - } - #containerKey(id) { - let key = this.#containerKeys.get(id); - if (key === undefined) { - key = formatContainerId(id); - this.#containerKeys.set(id, key); - } - return key; - } - #ensurePending() { - if (this.#pending !== undefined) - return this.#pending; - this.#materializeDeferredHistory(); - if (this.#detached && !this.#detachedEditing) { - throw new Error("cannot edit a detached document; call attach() first"); - } - const from = this.#frontiersCodec(); - const lamport = from.reduce((max, dependency) => Math.max(max, this.#lamportAt(dependency) + 1), 0); - this.#nextCounter = Math.max(this.#nextCounter, this.oplogVersion().get(this.#peer) ?? 0); - this.#pending = { - id: { peer: this.#peer, counter: this.#nextCounter }, - dependencies: from.map((id) => ({ ...id })), - lamport, - from, - operations: [], - operationLength: 0, - causalVersion: this.#causalVersionAt(from), - keys: [], - keyIndices: new Map(), - changedContainers: new Set(), - beforeValues: new Map(), - eventStates: new Map(), - }; - return this.#pending; - } - #nextOperationCounter(pending) { - return pending.id.counter + pending.operationLength; - } - #appendAndApply(container, content, length, preferredChild) { - if (container._codecId === undefined || container._doc !== this) - throw new Error("container is detached"); - const pending = this.#ensurePending(); - const counter = this.#nextOperationCounter(pending); - const operation = { - container: container._codecId, - counter, - length, - content, - }; - if (preferredChild !== undefined) { - const childId = this.#childIdForOperation(operation, preferredChild.kind()); - preferredChild._attach(this, childId, container); - this.#containers.set(preferredChild.id, preferredChild); - } - if (!appendToTrailingListInsert(pending.operations, operation)) { - pending.operations.push(operation); - } - pending.operationLength += operation.length; - pending.changedContainers.add(container.id); - const causalVersion = pending.causalVersion; - causalVersion.set(pending.id.peer, Math.max(causalVersion.get(pending.id.peer) ?? 0, operation.counter)); - const finishEvent = this.#prepareEvent(pending, container, operation, pending.keys, pending.id, causalVersion); - this.#applyOperation(operation, pending.keys, pending.id, pending.lamport, causalVersion, container); - finishEvent?.(); - return operation; - } - #prepareEvent(recording, container, operation, keys, changeId, causalVersion, force = false) { - if (!force && !this.#hasEventSubscribers()) - return undefined; - const content = operation.content; - if (container instanceof LoroText) { - const state = this.#sequenceEventState(recording, container, "text"); - if (content.type === "text-insert") { - return () => { - const first = container._sequence.findById({ - peer: changeId.peer, - counter: operation.counter, - }); - if (first === undefined || first.deleted) - return; - const position = container._sequence.visibleMetricOffsetOf(first, "utf16"); - if (position === undefined) - return; - const attributes = Object.fromEntries([...container._attributesAt(first)].map(([key, value]) => [ - key, - runtimeValueToJson(value), - ])); - state.diff.insertText(position, content.value, attributes); - }; - } - if (content.type === "text-delete") { - for (const range of this.#sequenceEventDeletionRanges(container, content.startId, Math.abs(Number(content.length))).reverse()) { - state.diff.delete(range.position, range.length); - } - return undefined; - } - if (content.type === "text-mark") { - const value = this.#decodeRuntimeValue(content.value, keys, { peer: changeId.peer, counter: operation.counter }, container); - const runs = container._styleRuns(content.start, content.end, causalVersion); - for (const range of container._sequence.visibleMetricRangesForIdRuns(runs, "utf16")) { - state.diff.formatText(range.start, range.end - range.start, content.key, runtimeValueToJson(value)); - } - return undefined; - } - if (content.type === "text-mark-end") - return undefined; - } - else if (container instanceof LoroList) { - const state = this.#sequenceEventState(recording, container, "list"); - if (content.type === "list-insert" || content.type === "movable-list-insert") { - return () => { - const first = container._sequence.findById({ - peer: changeId.peer, - counter: operation.counter, - }); - if (first === undefined || first.deleted) - return; - const position = container._sequence.visibleIndexOf(first); - if (position === undefined) - return; - state.diff.insertList(position, container - ._visibleElementsRange(position, position + content.values.length) - .map((element) => cloneRuntimeValue(element.value))); - }; - } - if (content.type === "list-delete" || content.type === "movable-list-delete") { - for (const range of this.#sequenceEventDeletionRanges(container, content.startId, Math.abs(Number(content.length))).reverse()) { - state.diff.delete(range.position, range.length); - } - return undefined; - } - if (content.type === "movable-list-move") { - const element = container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - const from = element === undefined || element.deleted - ? undefined - : container._sequence.visibleIndexOf(element); - if (element === undefined || from === undefined) - return undefined; - const value = cloneRuntimeValue(element.value); - return () => { - const to = element.deleted - ? undefined - : container._sequence.visibleIndexOf(element); - if (to === undefined || to === from) - return; - state.diff.delete(from, 1); - state.diff.insertList(to, [value]); - }; - } - if (content.type === "movable-list-set") { - const element = container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - const position = element === undefined || element.deleted - ? undefined - : container._sequence.visibleIndexOf(element); - if (element === undefined || position === undefined) - return undefined; - const selected = element; - const previous = cloneRuntimeValue(element.value); - return () => { - const value = cloneRuntimeValue(selected.value); - if (eventValuesEqual(previous, value)) - return; - state.diff.delete(position, 1); - state.diff.insertList(position, [value]); - }; - } - } - else if (container instanceof LoroMap) { - if (content.type === "map-insert" || content.type === "map-delete") { - const state = this.#mapEventState(recording, container); - const beforeRecord = container._entries.get(content.key); - const beforeVisible = beforeRecord !== undefined && !beforeRecord.deleted; - const beforeValue = beforeVisible - ? cloneRuntimeValue(beforeRecord.value) - : undefined; - if (!state.originals.has(content.key)) { - state.originals.set(content.key, { - present: beforeRecord !== undefined, - visible: beforeVisible, - value: beforeValue, - }); - } - return () => { - const afterRecord = container._entries.get(content.key); - const afterVisible = afterRecord !== undefined && !afterRecord.deleted; - const afterValue = afterVisible - ? cloneRuntimeValue(afterRecord.value) - : undefined; - if (beforeVisible !== afterVisible || - !eventValuesEqual(beforeValue, afterValue)) { - state.changed.add(content.key); - } - }; - } - } - else if (container instanceof LoroTree) { - if (content.type === "tree-create" || - content.type === "tree-move" || - content.type === "tree-delete") { - const state = this.#treeEventState(recording, container); - const id = formatTreeId(content.subject); - if (!state.originals.has(id)) { - const record = container._nodes.get(id); - state.originals.set(id, record === undefined || container._isRecordDeleted(record) - ? undefined - : this.#treeEventNode(container, record)); - } - return undefined; - } - } - else if (container instanceof LoroCounter) { - if (!recording.eventStates.has(container.id)) { - recording.eventStates.set(container.id, { - kind: "counter", - before: container.value, - }); - } - return undefined; - } - if (!recording.beforeValues.has(container.id)) { - recording.beforeValues.set(container.id, containerEventValue(container)); - } - return undefined; - } - #sequenceEventState(recording, container, kind) { - const existing = recording.eventStates.get(container.id); - if (existing !== undefined) { - if (existing.kind !== "sequence") - throw new Error("event state kind changed"); - return existing; - } - const state = { - kind: "sequence", - diff: new SequenceEventDiff(kind, container.length), - }; - recording.eventStates.set(container.id, state); - return state; - } - #mapEventState(recording, container) { - const existing = recording.eventStates.get(container.id); - if (existing !== undefined) { - if (existing.kind !== "map") - throw new Error("event state kind changed"); - return existing; - } - const state = { - kind: "map", - originals: new Map(), - changed: new Set(), - }; - recording.eventStates.set(container.id, state); - return state; - } - #treeEventState(recording, container) { - const existing = recording.eventStates.get(container.id); - if (existing !== undefined) { - if (existing.kind !== "tree") - throw new Error("event state kind changed"); - return existing; - } - const state = { - kind: "tree", - originals: new Map(), - }; - recording.eventStates.set(container.id, state); - return state; - } - #treeEventNode(tree, record) { - return { - id: formatTreeId(record.id), - parent: record.parent === undefined ? undefined : formatTreeId(record.parent), - index: tree._indexOf(record), - fractionalIndex: bytesToHex(record.position).toUpperCase(), - }; - } - #sequenceEventDeletionRanges(container, startId, length) { - return container._sequence - .visibleMetricRangesForIdRuns([{ start: { peer: startId.peer, counter: startId.counter }, length }], "utf16") - .map(({ start, end }) => ({ position: start, length: end - start })); - } - #recordedEventDiffs(recording, includeMapTombstones = false) { - const diffs = new Map(); - for (const [id, state] of recording.eventStates) { - const container = this.#containers.get(id); - if (container === undefined) - continue; - if (state.kind === "sequence") { - diffs.set(id, state.diff.toDiff()); - } - else if (state.kind === "map" && container instanceof LoroMap) { - const updated = []; - for (const [key, original] of state.originals) { - const record = container._entries.get(key); - const present = record !== undefined; - const visible = record !== undefined && !record.deleted; - const value = visible ? cloneRuntimeValue(record.value) : undefined; - if (state.changed.has(key) || - (includeMapTombstones && present !== original.present) || - visible !== original.visible || - !eventValuesEqual(value, original.value)) { - updated.push([key, value]); - } - } - diffs.set(id, { type: "map", updated: Object.fromEntries(updated) }); - } - else if (state.kind === "counter" && container instanceof LoroCounter) { - diffs.set(id, { type: "counter", increment: container.value - state.before }); - } - else if (state.kind === "tree" && container instanceof LoroTree) { - const before = []; - const after = []; - for (const [nodeId, original] of state.originals) { - if (original !== undefined) - before.push(original); - const record = container._nodes.get(nodeId); - if (record !== undefined && !container._isRecordDeleted(record)) { - after.push(this.#treeEventNode(container, record)); - } - } - diffs.set(id, { type: "tree", diff: treeDelta(before, after) }); - } - } - return diffs; - } - #attachChild(parent, child, content) { - if (child._doc !== undefined && child._doc !== this) { - throw new Error("cannot attach a container from another document"); - } - const blueprint = captureBlueprint(child); - const attached = createContainer(child.kind()); - const rawType = containerTypeToRawByte(publicTypeToCodec(child.kind())); - this.#appendAndApply(parent, content(rawType), 1, attached); - restoreBlueprint(attached, blueprint); - if (!child.isAttached()) - child._attached = attached; - return attached; - } - #childIdForOperation(operation, type) { - return { - kind: "normal", - peer: this.#peer, - counter: operation.counter, - containerType: publicTypeToCodec(type), - }; - } - #deleteSequenceRuns(container, position, length, type) { - for (const run of container._sequence.visibleIdRuns(position, position + length)) { - this.#appendAndApply(container, { type, position, length: BigInt(run.length), startId: run.start }, run.length); - } - } - #deleteTextRuns(container, position, length) { - for (const run of container._sequence.visibleIdRuns(position, position + length)) { - this.#appendAndApply(container, { - type: "text-delete", - position, - length: BigInt(run.length), - startId: run.start, - }, run.length); - this.#mergeTrailingTextDeletes(); - } - } - #mergeTrailingTextDeletes() { - const operations = this.#pending?.operations; - if (operations === undefined || operations.length < 2) - return; - const left = operations[operations.length - 2]; - const right = operations[operations.length - 1]; - if (left.content.type !== "text-delete" || - right.content.type !== "text-delete" || - !containerIdsEqual(left.container, right.container)) { - return; - } - const leftLength = Number(left.content.length); - const rightLength = Number(right.content.length); - if (rightLength <= 0 || left.content.startId.peer !== right.content.startId.peer) { - return; - } - let mergedLength; - let startId = left.content.startId; - if (leftLength > 0 && - left.content.position === right.content.position && - right.content.startId.counter === left.content.startId.counter + leftLength) { - mergedLength = leftLength + rightLength; - } - else if (leftLength === 1 && - right.content.position + rightLength === left.content.position && - right.content.startId.counter + rightLength === left.content.startId.counter) { - mergedLength = -(leftLength + rightLength); - startId = right.content.startId; - } - else if (leftLength < 0 && - right.content.position + rightLength - 1 === left.content.position + leftLength && - right.content.startId.counter + rightLength === left.content.startId.counter) { - mergedLength = leftLength - rightLength; - startId = right.content.startId; - } - if (mergedLength === undefined) - return; - operations.splice(operations.length - 2, 2, { - ...left, - length: left.length + right.length, - content: { - ...left.content, - length: BigInt(mergedLength), - startId, - }, - }); - } - #encodeRuntimeValue(value, pending) { - if (value == null) - return { type: "null" }; - if (typeof value === "boolean") - return { type: "bool", value }; - if (typeof value === "number") { - if (!Number.isFinite(value)) - throw new TypeError("Loro numbers must be finite"); - return Number.isSafeInteger(value) - ? { type: "i64", value: BigInt(value) } - : { type: "double", value }; - } - if (typeof value === "string") - return { type: "string", value }; - if (value instanceof Uint8Array) - return { type: "binary", value: value.slice() }; - if (Array.isArray(value)) { - return { - type: "list", - value: value.map((item) => this.#encodeRuntimeValue(item, pending)), - }; - } - if (isContainer(value)) - throw new TypeError("attach child containers with a container-specific method"); - if (typeof value === "object") { - if (Object.getOwnPropertySymbols(value).length > 0) { - throw new TypeError("Object keys must be strings"); - } - return { - type: "map", - value: Object.entries(value).map(([key, item]) => [ - BigInt(registerKey(pending, key)), - this.#encodeRuntimeValue(item, pending), - ]), - }; - } - throw new TypeError(`unsupported Loro value type: ${typeof value}`); - } - #decodeRuntimeValue(value, keys, operationId, parent) { - switch (value.type) { - case "null": - return null; - case "bool": - return value.value; - case "i64": - return Number(value.value); - case "double": - return value.value; - case "string": - return value.value; - case "binary": - return value.value.slice(); - case "list": - return value.value.map((item, index) => this.#decodeRuntimeValue(item, keys, { peer: operationId.peer, counter: operationId.counter + index }, parent)); - case "map": - return Object.fromEntries(value.value.map(([keyIndex, item]) => { - const key = keys[Number(keyIndex)]; - if (key === undefined) - throw new Error("change value map key index is out of range"); - return [key, this.#decodeRuntimeValue(item, keys, operationId, parent)]; - })); - case "container-type": { - const containerType = containerTypeFromRawByte(value.value); - const id = { kind: "normal", ...operationId, containerType }; - return this.#getOrCreateContainer(id, parent); - } - } - } - #applyOperation(operation, keys, changeId, changeLamport, causalVersion, knownContainer) { - const container = knownContainer ?? this.#getOrCreateContainer(operation.container); - const operationId = { peer: changeId.peer, counter: operation.counter }; - const lamport = changeLamport + (operation.counter - changeId.counter); - const content = operation.content; - switch (content.type) { - case "map-insert": - { - const rawValue = this.#decodeRuntimeValue(content.value, keys, operationId, container); - const writer = { peer: changeId.peer, lamport }; - container._applyValue(content.key, this.#materializeMapValue(container, content.key, rawValue), writer, rawValue); - } - return; - case "map-delete": - container._applyDelete(content.key, { - peer: changeId.peer, - lamport, - }); - return; - case "list-insert": - case "movable-list-insert": { - const values = content.values.map((value, index) => this.#decodeRuntimeValue(value, keys, { peer: operationId.peer, counter: operationId.counter + index }, container)); - container._insertFugue(content.position, values, values.map((_, index) => ({ - peer: operationId.peer, - counter: operationId.counter + index, - })), values.map((_, index) => lamport + index), causalVersion); - return; - } - case "list-delete": - case "movable-list-delete": - container._deleteIdSpan(content.startId, Number(content.length), operationId); - return; - case "text-insert": { - container._insertFugue(content.position, content.value, operationId, lamport, causalVersion); - return; - } - case "text-delete": - container._deleteIdSpan(content.startId, Number(content.length), operationId); - return; - case "text-mark": - { - const value = this.#decodeRuntimeValue(content.value, keys, operationId, container); - const meta = { - startId: operationId, - lamport, - info: content.info, - value, - }; - container._applyMark(content.start, content.end, content.key, value, meta, causalVersion); - } - return; - case "text-mark-end": - return; - case "movable-list-move": { - const list = container; - const element = list._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - const from = element === undefined || element.deleted - ? undefined - : list._sequence.visibleIndexOf(element); - if (from !== undefined) - list._applyMove(from, Math.min(content.to, list.length - 1), { - id: operationId, - lamport, - }); - return; - } - case "movable-list-set": { - const list = container; - const element = list._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - if (element !== undefined) { - const value = this.#decodeRuntimeValue(content.value, keys, operationId, container); - list._applySet(element, value, { - id: operationId, - lamport, - value, - }); - } - return; - } - case "tree-create": - case "tree-move": { - const tree = container; - const writer = { peer: changeId.peer, lamport }; - const nodeKey = formatTreeId(content.subject); - let record = tree._nodes.get(nodeKey); - if (record === undefined) { - const dataId = { - kind: "normal", - ...content.subject, - containerType: CodecContainerType.Map, - }; - const data = this.#getOrCreateContainer(dataId, tree); - record = { - id: content.subject, - parent: content.parent, - position: content.position.slice(), - deleted: false, - writer, - lastMoveId: operationId, - data, - }; - tree._setRecord(record); - } - else if (compareWriter(record.writer, writer) <= 0) { - tree._updateRecord(record, content.parent, content.position.slice(), writer, operationId); - } - return; - } - case "tree-delete": { - const record = container._nodes.get(formatTreeId(content.subject)); - if (record !== undefined) { - const writer = { peer: changeId.peer, lamport }; - if (compareWriter(record.writer, writer) <= 0) { - container._deleteRecord(record, writer, operationId); - } - } - return; - } - case "future": - if (container instanceof LoroCounter) { - if (content.value.type === "double" || content.value.type === "i64") { - container._value += - content.value.type === "double" - ? content.value.value - : Number(content.value.value); - } - else if (content.value.type === "delta-int") { - container._value += content.value.value; - } - } - } - } - #readChangeBlock(bytes) { - const block = decodeChangeBlock(bytes); - return block.changes.map((change) => ({ - change, - keys: block.keys, - keyIndices: undefined, - })); - } - #decodeImportData(parsed) { - if (parsed.mode === EncodeMode.FastUpdates) { - return { - mode: parsed.mode, - records: decodeFastUpdatesBody(parsed.body).flatMap((block) => this.#readChangeBlock(block)), - }; - } - const snapshot = decodeFastSnapshotBody(parsed.body); - const records = []; - let startVersion; - let startFrontiers; - let endVersion; - for (const entry of decodeSstable(snapshot.oplog)) { - if (bytesEqual(entry.key, VERSION_KEY)) { - endVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); - } - else if (bytesEqual(entry.key, START_VERSION_KEY)) { - startVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); - } - else if (bytesEqual(entry.key, START_FRONTIERS_KEY)) { - startFrontiers = decodePostcardFrontiers(entry.value); - } - else if (entry.key.length === 12) { - const expected = decodeChangeBlockKey(entry.key); - const block = this.#readChangeBlock(entry.value); - if (block[0] !== undefined && !idsEqual(block[0].change.id, expected)) { - throw new Error("snapshot change key does not match its block"); - } - records.push(...block); - } - } - return { - mode: parsed.mode, - records, - snapshot, - ...(startVersion === undefined ? {} : { startVersion }), - ...(startFrontiers === undefined ? {} : { startFrontiers }), - ...(endVersion === undefined ? {} : { endVersion }), - }; - } - #integrateHistory(records, mergeInterval = 0n) { - const added = []; - const addedRecordIndices = new Map(); - for (const record of records) { - const key = changeKey(record.change.id); - const knownEnd = Math.max(this.#shallowStartVersion.get(record.change.id.peer) ?? 0, this.#historyEndByPeer.get(record.change.id.peer) ?? 0); - const recordEnd = record.change.id.counter + changeLength(record.change); - if (recordEnd <= knownEnd) - continue; - const pending = this.#pendingHistory.get(key); - if (pending !== undefined && - pending.change.id.counter + changeLength(pending.change) >= recordEnd) - continue; - this.#pendingHistory.set(key, record); - } - // Hoist the sorted pending queue and the causal version out of the - // promotion loop; the version is advanced locally as records promote and - // stays in sync with #historyEndByPeer. The queue is compacted in place - // after each pass, and only rebuilt from the map when a sliced record was - // re-keyed (which may have overwritten another pending entry). - const version = this.#historyVersion(); - let queue = [...this.#pendingHistory.values()].sort(compareHistoryRecords); - let promoted = true; - while (promoted) { - promoted = false; - let rebuildQueue = false; - let kept = 0; - for (let index = 0; index < queue.length; index += 1) { - let record = queue[index]; - let change = record.change; - let key = changeKey(change.id); - const knownEnd = version.get(change.id.peer) ?? 0; - const changeEnd = change.id.counter + changeLength(change); - if (change.id.counter < knownEnd) { - this.#pendingHistory.delete(key); - if (changeEnd <= knownEnd) { - promoted = true; - continue; - } - record = { - ...record, - change: sliceChange(change, knownEnd - change.id.counter, changeLength(change)), - }; - change = record.change; - key = changeKey(change.id); - this.#pendingHistory.set(key, record); - promoted = true; - rebuildQueue = true; - } - if (change.id.counter !== (version.get(change.id.peer) ?? 0)) { - queue[kept] = record; - kept += 1; - continue; - } - if (change.dependencies.some((dependency) => !this.#historyContainsId(dependency))) { - queue[kept] = record; - kept += 1; - continue; - } - this.#pendingHistory.delete(key); - const previous = mergeInterval === undefined - ? undefined - : this.#mergeablePreviousRecord(change, mergeInterval); - if (previous === undefined) { - this.#setHistoryRecord(key, record, record); - addedRecordIndices.set(record, added.length); - } - else { - const addedPreviousIndex = addedRecordIndices.get(previous); - if (addedPreviousIndex !== undefined) { - added[addedPreviousIndex] = cloneHistoryRecord(previous); - addedRecordIndices.delete(previous); - } - const previousLength = changeLength(previous.change); - const appended = appendHistoryRecord(previous, record, previousLength); - this.#appendMergedHistoryRecord(previous, appended, previousLength); - } - version.set(change.id.peer, change.id.counter + changeLength(change)); - added.push(record); - this.#seenCommittedPeers.add(change.id.peer); - promoted = true; - } - queue.length = kept; - if (rebuildQueue) { - queue = [...this.#pendingHistory.values()].sort(compareHistoryRecords); - } - } - return { - added: added.sort(compareHistoryRecords), - pending: [...this.#pendingHistory.values()].sort(compareHistoryRecords), - }; - } - #historyContainsId(id) { - if (id.counter < (this.#shallowStartVersion.get(id.peer) ?? 0)) - return true; - return id.counter < (this.#historyEndByPeer.get(id.peer) ?? 0); - } - #assertImportsNotOutdated(records) { - if (this.#shallowRootStore === undefined) - return; - for (const { change } of records) { - const knownEnd = this.#shallowStartVersion.get(change.id.peer) ?? 0; - if (change.id.counter + changeLength(change) <= knownEnd) - continue; - if (change.dependencies.length === 0) { - throw new Error("cannot import updates that depend on an outdated version"); - } - const touchesShallowRoot = change.dependencies.some((dependency) => this.#shallowRootFrontiers.some((frontier) => idsEqual(frontier, dependency))); - if (touchesShallowRoot) - continue; - const dependsOnPrunedHistory = change.dependencies.some((dependency) => dependency.counter < (this.#shallowStartVersion.get(dependency.peer) ?? 0)); - if (dependsOnPrunedHistory) { - throw new Error("cannot import updates that depend on an outdated version"); - } - } - } - #applyRecords(records, recording) { - for (const record of records) { - const causalVersion = this.#causalVersionAt(record.change.dependencies); - for (const operation of record.change.operations) { - const container = this.#getOrCreateContainer(operation.container); - const finishEvent = recording === undefined - ? undefined - : this.#prepareEvent(recording, container, operation, record.keys, record.change.id, causalVersion, true); - this.#applyOperation(operation, record.keys, record.change.id, record.change.lamport, causalVersion, container); - finishEvent?.(); - causalVersion.set(record.change.id.peer, Math.max(causalVersion.get(record.change.id.peer) ?? 0, operation.counter + operation.length)); - } - } - } - #canTransitionRecords(records, movableMoveMode = "anchors") { - const replayedMoveContainers = new Set(); - if (movableMoveMode === "replay") { - for (const { change } of records) { - for (const operation of change.operations) { - if (operation.content.type === "movable-list-move") { - const containerId = this.#containerKey(operation.container); - replayedMoveContainers.add(containerId); - } - } - } - for (const containerId of replayedMoveContainers) { - if (this.#movableOrderHistory.get(containerId) === undefined) - return false; - } - } - const moveSuffixes = new Map(); - for (const { change } of records) { - const causalVersion = this.#causalVersionAt(change.dependencies); - for (const operation of change.operations) { - const container = this.#containers.get(this.#containerKey(operation.container)); - const content = operation.content; - if (content.type === "movable-list-move") { - const element = container instanceof LoroMovableList - ? container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport) - : undefined; - const history = element?.moveHistory; - const moveIndex = findSequenceMoveMetaIndex(history, operationWriter(change, operation)); - if (!(container instanceof LoroMovableList) || - !container._moveHistoryComplete || - element === undefined || - (movableMoveMode === "anchors" && - (history === undefined || - moveIndex < 0 || - history[moveIndex].id.peer !== change.id.peer || - history[moveIndex].id.counter !== operation.counter))) { - return false; - } - if (movableMoveMode === "anchors") { - let suffix = moveSuffixes.get(element); - if (suffix === undefined) { - suffix = { history: history, indices: new Set() }; - moveSuffixes.set(element, suffix); - } - suffix.indices.add(moveIndex); - } - } - else if (content.type === "movable-list-set") { - const element = container instanceof LoroMovableList - ? container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport) - : undefined; - if (!(container instanceof LoroMovableList) || - !container._valueHistoryComplete || - element === undefined || - !hasSequenceValueMeta(element.valueHistory, operationWriter(change, operation), change.id.peer, operation.counter)) { - return false; - } - } - else if (content.type === "text-mark") { - if (!(container instanceof LoroText) || !container._attributeHistoryComplete) { - return false; - } - const viewLength = container._sequence.isFullyIncluded(causalVersion) - ? container._sequence.visibleLength - : container._sequence.causalView(causalVersion).length; - if (content.end > viewLength) { - return false; - } - const runs = container._styleRuns(content.start, content.end, causalVersion); - if (!container._styleIndex.runsContainMeta(runs, content.key, { - peer: change.id.peer, - counter: operation.counter, - })) { - return false; - } - } - else if (content.type === "text-mark-end") { - if (!(container instanceof LoroText)) - return false; - } - else if (content.type === "text-insert") { - if (!(container instanceof LoroText)) - return false; - scratchSequenceIdRun.start.peer = change.id.peer; - scratchSequenceIdRun.start.counter = operation.counter; - scratchSequenceIdRun.length = operation.length; - if (!container._sequence.containsIdRuns(scratchSequenceIdRuns)) { - return false; - } - } - else if (content.type === "list-insert" || - content.type === "movable-list-insert") { - if (!(container instanceof LoroList)) - return false; - scratchSequenceIdRun.start.peer = change.id.peer; - scratchSequenceIdRun.start.counter = operation.counter; - scratchSequenceIdRun.length = operation.length; - if (!container._sequence.containsIdRuns(scratchSequenceIdRuns)) { - return false; - } - } - else if (content.type === "text-delete" || - content.type === "list-delete" || - content.type === "movable-list-delete") { - if (!(container instanceof LoroList || container instanceof LoroText)) { - return false; - } - } - else if (content.type === "map-insert" || content.type === "map-delete") { - if (!(container instanceof LoroMap)) - return false; - } - else if (content.type === "tree-create" || - content.type === "tree-move" || - content.type === "tree-delete") { - if (!(container instanceof LoroTree)) - return false; - } - else if (content.type === "future") { - if (!(container instanceof LoroCounter) || - counterDelta(content) === undefined) { - return false; - } - } - causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); - } - } - for (const { history, indices } of moveSuffixes.values()) { - const first = Math.min(...indices); - if (history.length - first !== indices.size) - return false; - for (let index = first; index < history.length; index += 1) { - if (!indices.has(index)) - return false; - } - } - return true; - } - #applyVersionTransition(retreat, forward, target, recording, movableMoveMode = "anchors") { - const sequences = new Map(); - const bulkSequenceRemovals = new Map(); - const bulkSequenceRestorations = new Map(); - const textAttributeRuns = new Map(); - const textStyleContainers = new Set(); - const movableValues = new Map(); - const mapKeys = new Map(); - const treeSubjects = new Map(); - const counters = new Map(); - const sequenceElements = (container) => { - let elements = sequences.get(container); - if (elements === undefined) { - elements = new Set(); - sequences.set(container, elements); - } - return elements; - }; - const addTextAttributeRuns = (container, key, runs) => { - let keys = textAttributeRuns.get(container); - if (keys === undefined) { - keys = new Map(); - textAttributeRuns.set(container, keys); - } - const existing = keys.get(key); - if (existing === undefined) - keys.set(key, [...runs]); - else - existing.push(...runs); - }; - const addBulkSequenceRemovals = (container, runs) => { - const existing = bulkSequenceRemovals.get(container); - if (existing === undefined) - bulkSequenceRemovals.set(container, [...runs]); - else - existing.push(...runs); - }; - const addBulkSequenceRestorations = (container, runs) => { - const existing = bulkSequenceRestorations.get(container); - if (existing === undefined) - bulkSequenceRestorations.set(container, [...runs]); - else - existing.push(...runs); - }; - const collect = (records, direction) => { - for (const { change } of records) { - const causalVersion = this.#causalVersionAt(change.dependencies); - for (const operation of change.operations) { - const container = this.#containers.get(this.#containerKey(operation.container)); - const content = operation.content; - if (content.type === "text-insert" || - content.type === "list-insert" || - content.type === "movable-list-insert") { - const sequenceContainer = container; - const sequence = sequenceContainer._sequence; - scratchSequenceIdRun.start.peer = change.id.peer; - scratchSequenceIdRun.start.counter = operation.counter; - scratchSequenceIdRun.length = operation.length; - if (direction === -1 || - (recording === undefined && - sequence.canShowIdRunsAt(scratchSequenceIdRuns, target))) { - // The bulk lists retain their runs, so they get a fresh copy - // rather than the shared scratch run. - const insertedRuns = [ - { - start: { peer: change.id.peer, counter: operation.counter }, - length: operation.length, - }, - ]; - if (direction === -1) { - addBulkSequenceRemovals(sequenceContainer, insertedRuns); - } - else { - addBulkSequenceRestorations(sequenceContainer, insertedRuns); - } - causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); - continue; - } - const elements = sequenceElements(sequenceContainer); - for (let offset = 0; offset < operation.length; offset += 1) { - elements.add(sequence.findById({ - peer: change.id.peer, - counter: operation.counter + offset, - })); - } - } - else if (content.type === "text-delete" || - content.type === "list-delete" || - content.type === "movable-list-delete") { - const sequenceContainer = container; - const deletedRuns = sequenceContainer._sequence.idRunsDeletedBy(change.id.peer, operation.counter, operation.counter + operation.length); - if (direction === 1 || - (recording === undefined && - sequenceContainer._sequence.canShowIdRunsAt(deletedRuns, target))) { - if (direction === 1) { - addBulkSequenceRemovals(sequenceContainer, deletedRuns); - } - else { - addBulkSequenceRestorations(sequenceContainer, deletedRuns); - } - causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); - continue; - } - const elements = sequenceElements(sequenceContainer); - for (const element of sequenceContainer._sequence.elementsDeletedBy(change.id.peer, operation.counter, operation.counter + operation.length)) { - elements.add(element); - } - } - else if (content.type === "text-mark") { - const text = container; - textStyleContainers.add(text); - if (recording !== undefined) { - addTextAttributeRuns(text, content.key, text._styleRuns(content.start, content.end, causalVersion)); - } - } - else if (content.type === "movable-list-set") { - const list = container; - const element = list._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - let elements = movableValues.get(list); - if (elements === undefined) { - elements = new Set(); - movableValues.set(list, elements); - } - elements.add(element); - } - else if (content.type === "map-insert" || content.type === "map-delete") { - const map = container; - let keys = mapKeys.get(map); - if (keys === undefined) { - keys = new Set(); - mapKeys.set(map, keys); - } - keys.add(content.key); - } - else if (content.type === "tree-create" || - content.type === "tree-move" || - content.type === "tree-delete") { - const tree = container; - let subjects = treeSubjects.get(tree); - if (subjects === undefined) { - subjects = new Map(); - treeSubjects.set(tree, subjects); - } - subjects.set(idKey(content.subject), content.subject); - } - else if (content.type === "future") { - const counter = container; - counters.set(counter, (counters.get(counter) ?? 0) + direction * counterDelta(content)); - } - causalVersion.set(change.id.peer, Math.max(causalVersion.get(change.id.peer) ?? 0, operation.counter + operation.length)); - } - } - }; - collect(retreat, -1); - collect(forward, 1); - if (recording !== undefined) { - for (const [container, runs] of [...bulkSequenceRemovals]) { - if (!sequences.has(container)) - continue; - const elements = sequenceElements(container); - for (const run of runs) { - for (let offset = 0; offset < run.length; offset += 1) { - const element = container._sequence.findById({ - peer: run.start.peer, - counter: run.start.counter + offset, - }); - if (element !== undefined) - elements.add(element); - } - } - bulkSequenceRemovals.delete(container); - } - } - const includes = (id) => id.counter < (target.get(id.peer) ?? 0); - const targetDeleted = (container, element) => !includes(element.id) || container._sequence.someDeletion(element, includes); - const styleVersion = target.compare(this.#historyVersion()) === 0 - ? undefined - : new Map(target - ._codecEntriesUnsorted() - .map(({ peer, counter }) => [peer, counter])); - for (const [text, keys] of textAttributeRuns) { - const state = this.#sequenceEventState(recording, text, "text"); - const removedRuns = [ - ...(bulkSequenceRemovals.get(text) ?? []), - ...[...(sequences.get(text) ?? [])] - .filter((element) => !element.deleted && targetDeleted(text, element)) - .map((element) => ({ start: element.id, length: 1 })), - ]; - for (const [key, runs] of keys) { - for (const transition of text._styleIndex.transitions(runs, key, text._styleVersion, styleVersion)) { - const beforePresent = transition.before !== undefined && transition.before.value !== null; - const beforeValue = beforePresent ? transition.before.value : undefined; - const afterPresent = transition.after !== undefined && transition.after.value !== null; - const afterValue = afterPresent ? transition.after.value : undefined; - if (beforePresent === afterPresent && - eventValuesEqual(beforeValue, afterValue)) { - continue; - } - const retainedRuns = subtractSequenceIdRuns([transition.run], removedRuns); - for (const range of text._sequence.visibleMetricRangesForIdRuns(retainedRuns, "utf16")) { - state.diff.formatText(range.start, range.end - range.start, key, afterPresent ? runtimeValueToJson(afterValue) : null); - } - } - } - } - for (const text of textStyleContainers) - text._setStyleVersion(styleVersion); - for (const [container, runs] of bulkSequenceRestorations) { - container._sequence.setIdRunsVisible(runs); - } - for (const [container, elements] of sequences) { - const state = recording === undefined - ? undefined - : this.#sequenceEventState(recording, container, container instanceof LoroText ? "text" : "list"); - const removals = [...elements] - .filter((element) => !element.deleted && targetDeleted(container, element)) - .map((element) => ({ - element, - position: container instanceof LoroText - ? container._sequence.visibleMetricOffsetOf(element, "utf16") - : container._sequence.visibleIndexOf(element), - })) - .sort((left, right) => right.position - left.position); - for (const { element, position } of removals) { - state?.diff.delete(position, container instanceof LoroText ? element.value.length : 1); - container._sequence.setDeleted(element, true); - } - const insertions = [...elements] - .filter((element) => element.deleted && !targetDeleted(container, element)) - .sort((left, right) => container._sequence.physicalIndexOf(left) - - container._sequence.physicalIndexOf(right)); - for (const element of insertions) { - container._sequence.setDeleted(element, false); - if (state === undefined) - continue; - if (container instanceof LoroText) { - const textElement = element; - const position = container._sequence.visibleMetricOffsetOf(textElement, "utf16"); - const attributes = Object.fromEntries([...container._attributesAt(textElement)].map(([key, value]) => [ - key, - runtimeValueToJson(value), - ])); - state.diff.insertText(position, textElement.value, attributes); - } - else { - const position = container._sequence.visibleIndexOf(element); - state.diff.insertList(position, [cloneRuntimeValue(element.value)]); - } - } - } - for (const [container, runs] of bulkSequenceRemovals) { - if (recording !== undefined) { - const state = this.#sequenceEventState(recording, container, container instanceof LoroText ? "text" : "list"); - for (const range of container._sequence - .visibleMetricRangesForIdRuns(runs, "utf16") - .reverse()) { - state.diff.delete(range.start, range.end - range.start); - } - } - container._sequence.setIdRunsDeleted(runs); - } - if (movableMoveMode === "replay") { - this.#replayMovableMoves(retreat, forward, target, recording); - } - else { - this.#transitionMovableMoves(retreat, true, recording); - this.#transitionMovableMoves(forward, false, recording); - } - for (const [list, elements] of movableValues) { - const state = recording === undefined - ? undefined - : this.#sequenceEventState(recording, list, "list"); - for (const element of elements) { - const winner = latestIncludedSequenceValue(element.valueHistory, target); - if (winner === undefined || eventValuesEqual(element.value, winner.value)) { - continue; - } - if (!element.deleted) { - const position = list._sequence.visibleIndexOf(element); - if (position !== undefined) { - state?.diff.delete(position, 1); - state?.diff.insertList(position, [cloneRuntimeValue(winner.value)]); - } - } - element.value = winner.value; - list._bindChildren([element]); - } - } - for (const [map, keys] of mapKeys) { - const history = this.#mapOperationHistory.get(map.id); - for (const key of keys) { - if (recording !== undefined) { - const state = this.#mapEventState(recording, map); - if (!state.originals.has(key)) { - const record = map._entries.get(key); - state.originals.set(key, { - present: record !== undefined, - visible: record !== undefined && !record.deleted, - value: record === undefined || record.deleted - ? undefined - : cloneRuntimeValue(record.value), - }); - } - } - const indexed = latestIncludedOperation(history?.get(key), target); - if (indexed === undefined) { - map._replaceRecord(key, undefined); - continue; - } - const content = indexed.operation.content; - if (content.type === "map-delete") { - map._replaceRecord(key, { - value: undefined, - rawValue: undefined, - deleted: true, - writer: indexed.writer, - }); - } - else if (content.type === "map-insert") { - const operationId = { - peer: indexed.record.change.id.peer, - counter: indexed.operation.counter, - }; - const rawValue = this.#decodeRuntimeValue(content.value, indexed.record.keys, operationId, map); - map._replaceRecord(key, { - value: this.#materializeMapValue(map, key, rawValue), - rawValue, - deleted: false, - writer: indexed.writer, - }); - } - } - } - if (recording !== undefined) { - for (const [tree, subjects] of treeSubjects) { - const state = this.#treeEventState(recording, tree); - for (const subject of subjects.values()) { - const nodeKey = formatTreeId(subject); - if (state.originals.has(nodeKey)) - continue; - const record = tree._nodes.get(nodeKey); - state.originals.set(nodeKey, record === undefined || tree._isRecordDeleted(record) - ? undefined - : this.#treeEventNode(tree, record)); - } - } - } - for (const [tree, subjects] of treeSubjects) { - const history = this.#treeOperationHistory.get(tree.id); - for (const [historyKey, subject] of subjects) { - const nodeKey = formatTreeId(subject); - const operations = history?.get(historyKey); - const winner = latestIncludedOperation(operations, target); - const existing = tree._nodes.get(nodeKey); - if (winner === undefined) { - if (existing !== undefined) - tree._removeRecord(existing); - continue; - } - const winnerContent = winner.operation.content; - const placement = winnerContent.type === "tree-create" || winnerContent.type === "tree-move" - ? winner - : latestIncludedTreePlacement(operations, target, winner.writer); - if (placement === undefined) - continue; - const placementContent = placement.operation.content; - if (placementContent.type !== "tree-create" && - placementContent.type !== "tree-move") { - continue; - } - let record = tree._nodes.get(nodeKey); - const lastMoveId = { - peer: placement.record.change.id.peer, - counter: placement.operation.counter, - }; - if (record === undefined) { - const data = this.#getOrCreateContainer({ - kind: "normal", - ...placementContent.subject, - containerType: CodecContainerType.Map, - }, tree); - record = { - id: placementContent.subject, - parent: placementContent.parent, - position: placementContent.position.slice(), - deleted: false, - writer: placement.writer, - lastMoveId, - data, - }; - tree._setRecord(record); - } - else { - tree._updateRecord(record, placementContent.parent, placementContent.position.slice(), placement.writer, lastMoveId); - } - if (winnerContent.type === "tree-delete") { - tree._deleteRecord(record, winner.writer, { - peer: winner.record.change.id.peer, - counter: winner.operation.counter, - }); - } - } - } - for (const [counter, adjustment] of counters) { - if (recording !== undefined && !recording.eventStates.has(counter.id)) { - recording.eventStates.set(counter.id, { - kind: "counter", - before: counter.value, - }); - } - counter._value += adjustment; - } - } - #transitionMovableMoves(records, retreat, recording) { - const operations = records - .flatMap(({ change }) => change.operations.flatMap((operation) => operation.content.type === "movable-list-move" ? [{ change, operation }] : [])) - .sort((left, right) => compareHistoryOperations(left.change, left.operation, right.change, right.operation)); - if (retreat) - operations.reverse(); - for (const { change, operation } of operations) { - const content = operation.content; - if (content.type !== "movable-list-move") - continue; - const container = this.#containers.get(this.#containerKey(operation.container)); - if (!(container instanceof LoroMovableList)) - continue; - const element = container._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - if (element === undefined || element.deleted) - continue; - const meta = findSequenceMoveMeta(element.moveHistory, operationWriter(change, operation)); - if (meta === undefined) - continue; - const from = container._sequence.visibleIndexOf(element); - if (from === undefined) - continue; - const value = cloneRuntimeValue(element.value); - container._moveToAnchors(element, retreat ? meta.beforePrevious : meta.afterPrevious, retreat ? meta.beforeNext : meta.afterNext); - const to = container._sequence.visibleIndexOf(element); - if (recording === undefined || to === undefined || to === from) - continue; - const state = this.#sequenceEventState(recording, container, "list"); - state.diff.delete(from, 1); - state.diff.insertList(to, [value]); - } - } - #canonicalizeImportedMovableMoves(records, recording) { - let hasMove = false; - for (const { change } of records) { - for (const operation of change.operations) { - if (operation.content.type !== "movable-list-move") - continue; - hasMove = true; - const container = this.#containers.get(this.#containerKey(operation.container)); - if (!(container instanceof LoroMovableList) || !container._moveHistoryComplete) { - return; - } - } - } - if (hasMove) - this.#replayMovableMoves([], records, this.#historyVersion(), recording); - } - #replayMovableMoves(retreat, forward, target, recording) { - const containerIds = new Set(); - for (const { change } of [...retreat, ...forward]) { - for (const operation of change.operations) { - if (operation.content.type === "movable-list-move") { - containerIds.add(this.#containerKey(operation.container)); - } - } - } - for (const containerId of containerIds) { - const container = this.#containers.get(containerId); - const history = this.#movableOrderHistory.get(containerId); - if (!(container instanceof LoroMovableList) || history === undefined) - continue; - const replay = new LoroMovableList(); - const ordered = history - .values() - .map((indexed) => ({ - indexed, - orderRecord: this.#recordContaining({ - peer: indexed.record.change.id.peer, - counter: indexed.operation.counter, - }) ?? indexed.record, - })) - .sort((left, right) => compareHistoryRecords(left.orderRecord, right.orderRecord) || - left.indexed.operation.counter - right.indexed.operation.counter); - for (const { indexed } of ordered) { - const peer = indexed.record.change.id.peer; - const includedLength = Math.min(indexed.operation.length, (target.get(peer) ?? 0) - indexed.operation.counter); - if (includedLength <= 0) - continue; - const operation = includedLength === indexed.operation.length - ? indexed.operation - : sliceOperation(indexed.operation, 0, includedLength); - const content = operation.content; - const operationId = { peer, counter: operation.counter }; - const lamport = indexed.record.change.lamport + - operation.counter - - indexed.record.change.id.counter; - const causalVersion = this.#causalVersionAt(indexed.record.change.dependencies); - causalVersion.set(peer, Math.max(causalVersion.get(peer) ?? 0, operation.counter)); - if (content.type === "movable-list-insert") { - replay._insertFugue(content.position, content.values.map(() => null), content.values.map((_, offset) => ({ - peer, - counter: operation.counter + offset, - })), content.values.map((_, offset) => lamport + offset), causalVersion); - } - else if (content.type === "movable-list-delete") { - replay._deleteIdSpan(content.startId, Number(content.length), operationId); - } - else if (content.type === "movable-list-move") { - const element = replay._sequence.findByLamport(content.elementId.peer, content.elementId.lamport); - const from = element === undefined || element.deleted - ? undefined - : replay._sequence.visibleIndexOf(element); - if (from !== undefined) { - replay._applyMove(from, Math.min(content.to, replay.length - 1), { - id: operationId, - lamport, - }); - } - } - } - const current = container._visibleElements(); - const currentIndex = new Map(current.map((element, index) => [element, index])); - const targetElements = replay._visibleElements().map((replayed) => { - const element = container._sequence.findById(replayed.id); - if (element === undefined || element.deleted) { - throw new Error("movable-list replay produced an unavailable target element"); - } - return element; - }); - for (const replayed of replay._elements) { - const element = container._sequence.findById(replayed.id); - if (element === undefined) - continue; - for (const meta of replayed.moveHistory ?? []) { - let history = element.moveHistory; - if (history === undefined) { - history = []; - element.moveHistory = history; - } - const existing = history.findIndex(({ id }) => id.peer === meta.id.peer && id.counter === meta.id.counter); - if (existing >= 0) - history[existing] = meta; - else { - history.push(meta); - history.sort((left, right) => compareWriter({ peer: left.id.peer, lamport: left.lamport }, { peer: right.id.peer, lamport: right.lamport })); - } - } - } - if (targetElements.length !== current.length || - targetElements.some((element) => !currentIndex.has(element))) { - throw new Error("movable-list replay changed the visible element set"); - } - const stableTargetIndices = longestIncreasingSubsequenceIndices(targetElements.map((element) => currentIndex.get(element))); - const state = recording === undefined - ? undefined - : this.#sequenceEventState(recording, container, "list"); - for (let index = targetElements.length - 1; index >= 0; index -= 1) { - if (stableTargetIndices.has(index)) - continue; - const element = targetElements[index]; - const from = container._sequence.visibleIndexOf(element); - container._sequence.moveBefore(element, targetElements[index + 1]); - const to = container._sequence.visibleIndexOf(element); - if (state !== undefined && from !== to) { - state.diff.delete(from, 1); - state.diff.insertList(to, [cloneRuntimeValue(element.value)]); - } - } - } - } - #rebuildFromHistory(version) { - for (const container of this.#containers.values()) - container._reset(); - if (this.#shallowRootStore !== undefined) { - this.#hydrateState(this.#shallowRootStore); - const target = version ?? this.#historyVersion(); - this.#assertVersionNotBeforeShallowRoot(target); - this.#applyRecords(this.#recordsInVersionRange(this.#shallowRootVersion, target)); - return; - } - this.#applyRecords(version === undefined ? this.#sortedHistory() : this.#recordsAtVersion(version)); - } - #setHistoryRecord(key, record, appended) { - const previous = this.#history.get(key); - this.#history.set(key, record); - if (previous !== undefined) - this.#historyOrder.delete(previous); - this.#historyOrder.add(record); - this.#historyOperationCount += - changeLength(record.change) - - (previous === undefined ? 0 : changeLength(previous.change)); - this.#sortedHistoryCache = undefined; - let peerRecords = this.#historyByPeer.get(record.change.id.peer); - if (peerRecords === undefined) { - peerRecords = []; - this.#historyByPeer.set(record.change.id.peer, peerRecords); - } - if (previous !== undefined) { - const previousIndex = lowerBoundHistory(peerRecords, previous.change.id.counter); - if (peerRecords[previousIndex] === previous) - peerRecords.splice(previousIndex, 1); - } - const index = lowerBoundHistory(peerRecords, record.change.id.counter); - peerRecords.splice(index, 0, record); - const last = peerRecords.at(-1); - if (last === undefined) { - this.#historyEndByPeer.delete(record.change.id.peer); - } - else { - this.#historyEndByPeer.set(record.change.id.peer, last.change.id.counter + changeLength(last.change)); - } - if (previous !== undefined) { - const previousLast = { - peer: previous.change.id.peer, - counter: previous.change.id.counter + changeLength(previous.change) - 1, - }; - this.#historyFrontiers.delete(idKey(previousLast)); - } - for (const dependency of appended.change.dependencies) { - this.#historyFrontiers.delete(idKey(dependency)); - } - const lastId = { - peer: record.change.id.peer, - counter: record.change.id.counter + changeLength(record.change) - 1, - }; - this.#historyFrontiers.set(idKey(lastId), lastId); - this.#dependencyVersion(record.change); - this.#indexHistoryOperations(appended); - } - #appendMergedHistoryRecord(record, appended, previousLength) { - const appendedLength = changeLength(appended.change); - this.#historyOperationCount += appendedLength; - this.#historyEndByPeer.set(record.change.id.peer, record.change.id.counter + previousLength + appendedLength); - const previousLast = { - peer: record.change.id.peer, - counter: record.change.id.counter + previousLength - 1, - }; - this.#historyFrontiers.delete(idKey(previousLast)); - for (const dependency of appended.change.dependencies) { - this.#historyFrontiers.delete(idKey(dependency)); - } - const lastId = { - peer: record.change.id.peer, - counter: record.change.id.counter + previousLength + appendedLength - 1, - }; - this.#historyFrontiers.set(idKey(lastId), lastId); - this.#indexHistoryOperations(appended); - } - #indexHistoryOperations(record) { - for (const operation of record.change.operations) { - this.#containersWithOperations.add(this.#containerKey(operation.container)); - const content = operation.content; - if (content.type === "movable-list-insert" || - content.type === "movable-list-delete" || - content.type === "movable-list-move") { - const indexed = { - record, - operation, - writer: operationWriter(record.change, operation), - }; - const container = this.#containerKey(operation.container); - let history = this.#movableOrderHistory.get(container); - if (history === undefined) { - history = new OrderedIndex((left, right) => compareWriter(left.writer, right.writer)); - this.#movableOrderHistory.set(container, history); - } - history.add(indexed); - if (content.type === "movable-list-move") { - let peers = this.#movableMovePeers.get(container); - if (peers === undefined) { - peers = new Set(); - this.#movableMovePeers.set(container, peers); - } - peers.add(record.change.id.peer); - } - } - let bySubject; - let subject; - let treeOperation = false; - if (content.type === "map-insert" || content.type === "map-delete") { - const container = this.#containerKey(operation.container); - bySubject = this.#mapOperationHistory.get(container); - if (bySubject === undefined) { - bySubject = new Map(); - this.#mapOperationHistory.set(container, bySubject); - } - subject = content.key; - } - else if (content.type === "tree-create" || - content.type === "tree-move" || - content.type === "tree-delete") { - const container = this.#containerKey(operation.container); - bySubject = this.#treeOperationHistory.get(container); - if (bySubject === undefined) { - bySubject = new Map(); - this.#treeOperationHistory.set(container, bySubject); - } - subject = idKey(content.subject); - treeOperation = true; - } - if (bySubject === undefined || subject === undefined) - continue; - // Only movable/map/tree operations reach here, so the index entry and - // its writer are allocated off the text/list hot path. - const indexed = { - record, - operation, - writer: operationWriter(record.change, operation), - }; - let history = bySubject.get(subject); - if (history === undefined) { - history = { - byWriter: new OrderedIndex((left, right) => compareWriter(left.writer, right.writer)), - byPeer: new Map(), - ...(treeOperation ? { placementsByPeer: new Map() } : {}), - }; - bySubject.set(subject, history); - } - history.byWriter.add(indexed); - let peerOperations = history.byPeer.get(record.change.id.peer); - if (peerOperations === undefined) { - peerOperations = []; - history.byPeer.set(record.change.id.peer, peerOperations); - } - const peerIndex = lowerBoundIndexedOperation(peerOperations, operation.counter); - peerOperations.splice(peerIndex, 0, indexed); - if (content.type === "tree-create" || content.type === "tree-move") { - let placements = history.placementsByPeer.get(record.change.id.peer); - if (placements === undefined) { - placements = []; - history.placementsByPeer.set(record.change.id.peer, placements); - } - const placementIndex = lowerBoundIndexedOperation(placements, operation.counter); - placements.splice(placementIndex, 0, indexed); - } - } - } - #validateDeferredFrontierBlocks(entries, endVersion, frontiers) { - const changeEntriesByPeer = new Map(); - for (const entry of entries) { - if (entry.key.length !== 12) - continue; - const start = decodeChangeBlockKey(entry.key); - const peerEntries = changeEntriesByPeer.get(start.peer); - if (peerEntries === undefined) { - changeEntriesByPeer.set(start.peer, [{ entry, start }]); - } - else { - peerEntries.push({ entry, start }); - } - } - for (const peerEntries of changeEntriesByPeer.values()) { - peerEntries.sort((left, right) => left.start.counter - right.start.counter); - } - const seenPeers = new Set(); - for (const frontier of frontiers) { - const peerEnd = endVersion.get(frontier.peer) ?? 0; - if (frontier.counter >= peerEnd) { - throw new Error("snapshot frontier exceeds its end version"); - } - if (seenPeers.has(frontier.peer)) { - throw new Error("snapshot contains multiple frontiers for one peer"); - } - seenPeers.add(frontier.peer); - const peerEntries = changeEntriesByPeer.get(frontier.peer) ?? []; - let low = 0; - let high = peerEntries.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (peerEntries[middle].start.counter <= frontier.counter) - low = middle + 1; - else - high = middle; - } - const candidate = peerEntries[low - 1]; - if (candidate === undefined) { - throw new Error("snapshot frontier change block is missing"); - } - const block = validateChangeBlock(candidate.entry.value); - if (block.peer !== candidate.start.peer || - block.counterStart !== candidate.start.counter) { - throw new Error("snapshot change key does not match its block"); - } - if (block.peer !== frontier.peer || - frontier.counter < block.counterStart || - frontier.counter >= block.counterEnd) { - throw new Error("snapshot frontier is not covered by its change block"); - } - } - } - #materializeDeferredHistory() { - const deferred = this.#deferredSnapshotHistory; - if (deferred === undefined) - return; - const records = []; - for (const entry of deferred.entries) { - if (entry.key.length !== 12) - continue; - const expected = decodeChangeBlockKey(entry.key); - const decoded = this.#readChangeBlock(entry.value); - if (decoded[0] !== undefined && !idsEqual(decoded[0].change.id, expected)) { - throw new Error("snapshot change key does not match its block"); - } - records.push(...decoded.map(cloneHistoryRecord)); - } - // Validate and build every history index on an isolated document. A malformed - // deferred block must not leave this document with half-installed history. - const staged = new LoroDoc(); - staged.#shallowStartVersion = this.#shallowStartVersion.clone(); - const integration = staged.#integrateHistory(records); - if (integration.pending.length > 0) { - throw new Error("snapshot history contains changes with missing dependencies"); - } - if (staged.#historyVersion().compare(deferred.endVersion) !== 0) { - throw new Error("snapshot version does not match its history"); - } - const actualFrontiers = [...staged.#historyFrontiers.values()] - .sort(compareIds) - .map(formatOpId); - const expectedFrontiers = [...deferred.frontiers].sort(compareIds).map(formatOpId); - if (!frontierSetsEqual(actualFrontiers, expectedFrontiers)) { - throw new Error("snapshot frontiers do not match its history"); - } - if (staged.#historyOperationCount !== deferred.operationCount) { - throw new Error("snapshot operation count does not match its history"); - } - // State was already hydrated from the latest-state SSTable. Installing these - // structures restores only history/DAG indexes; applying records would - // duplicate counters and sequence content. - this.#history = staged.#history; - this.#historyOrder = staged.#historyOrder; - this.#historyByPeer = staged.#historyByPeer; - this.#historyEndByPeer = staged.#historyEndByPeer; - this.#historyOperationCount = staged.#historyOperationCount; - this.#sortedHistoryCache = staged.#sortedHistoryCache; - this.#historyFrontiers = staged.#historyFrontiers; - this.#dependencyVersionCache = staged.#dependencyVersionCache; - this.#mapOperationHistory = staged.#mapOperationHistory; - this.#treeOperationHistory = staged.#treeOperationHistory; - this.#movableOrderHistory = staged.#movableOrderHistory; - this.#movableMovePeers = staged.#movableMovePeers; - this.#containersWithOperations = staged.#containersWithOperations; - this.#containerKeys = staged.#containerKeys; - this.#pendingHistory = staged.#pendingHistory; - this.#deferredSnapshotHistory = undefined; - } - #sortedHistory() { - this.#materializeDeferredHistory(); - return (this.#sortedHistoryCache ??= this.#historyOrder.values()); - } - #recordsAtVersion(version) { - return this.#recordsInVersionRange(new VersionVector(), version); - } - #recordsInVersionRange(from, to) { - this.#materializeDeferredHistory(); - const selected = []; - for (const { peer, counter: toCounter } of to._codecEntriesUnsorted()) { - const fromCounter = from.get(peer) ?? 0; - if (fromCounter >= toCounter) - continue; - const records = this.#historyByPeer.get(peer) ?? []; - let index = Math.max(0, lowerBoundHistory(records, fromCounter + 1) - 1); - for (; index < records.length; index += 1) { - const record = records[index]; - const recordStart = record.change.id.counter; - if (recordStart >= toCounter) - break; - const length = changeLength(record.change); - const start = Math.max(0, fromCounter - recordStart); - const end = Math.min(length, toCounter - recordStart); - if (start >= end) - continue; - selected.push(start === 0 && end === length - ? record - : { ...record, change: sliceChange(record.change, start, end) }); - } - } - return selected.sort(compareHistoryRecords); - } - #recordsInSpans(spans) { - this.#materializeDeferredHistory(); - const spansByPeer = new Map(); - for (const { id, len } of spans) { - if (!Number.isSafeInteger(len) || len < 0) { - throw new RangeError(`update span length is out of range: ${len}`); - } - const parsed = parseOpId(id); - if (len === 0) - continue; - let peerSpans = spansByPeer.get(parsed.peer); - if (peerSpans === undefined) { - peerSpans = []; - spansByPeer.set(parsed.peer, peerSpans); - } - peerSpans.push({ start: parsed.counter, end: parsed.counter + len }); - } - const selected = []; - for (const [peer, ranges] of spansByPeer) { - ranges.sort((left, right) => left.start - right.start); - const merged = []; - for (const range of ranges) { - const previous = merged.length > 0 ? merged[merged.length - 1] : undefined; - if (previous !== undefined && range.start <= previous.end) { - previous.end = Math.max(previous.end, range.end); - } - else { - merged.push({ ...range }); - } - } - const records = this.#historyByPeer.get(peer) ?? []; - for (const range of merged) { - let index = Math.max(0, lowerBoundHistory(records, range.start + 1) - 1); - for (; index < records.length; index += 1) { - const record = records[index]; - const changeStart = record.change.id.counter; - if (changeStart >= range.end) - break; - const length = changeLength(record.change); - const start = Math.max(0, range.start - changeStart); - const end = Math.min(length, range.end - changeStart); - if (start >= end) - continue; - selected.push(start === 0 && end === length - ? record - : { ...record, change: sliceChange(record.change, start, end) }); - } - } - const preCommit = this.#preCommitRecord; - if (preCommit?.change.id.peer === peer) { - const changeStart = preCommit.change.id.counter; - const length = changeLength(preCommit.change); - const changeEnd = changeStart + length; - for (const range of merged) { - const start = Math.max(changeStart, range.start); - const end = Math.min(changeEnd, range.end); - if (start >= end) - continue; - selected.push(start === changeStart && end === changeEnd - ? preCommit - : { - ...preCommit, - change: sliceChange(preCommit.change, start - changeStart, end - changeStart), - }); - } - } - } - return selected.sort(compareHistoryRecords); - } - #assertVersionNotBeforeShallowRoot(version) { - if (!versionIncludes(version, this.#shallowRootVersion)) { - throw new RangeError("cannot use a version before the shallow history root"); - } - } - #causalVersionAt(frontiers) { - // Most documents have no shallow root; avoid the entry array and map - // copies in that case. - const version = this.#shallowStartVersion.length() === 0 - ? new Map() - : new Map(this.#shallowStartVersion - ._codecEntriesUnsorted() - .map(({ peer, counter }) => [peer, counter])); - for (const id of frontiers) { - const record = this.#recordContaining(id); - if (record !== undefined) { - for (const [peer, counter] of this.#dependencyVersion(record.change)) { - version.set(peer, Math.max(version.get(peer) ?? 0, counter)); - } - } - version.set(id.peer, Math.max(version.get(id.peer) ?? 0, id.counter + 1)); - } - return version; - } - #dependencyVersion(change) { - const cached = this.#dependencyVersionCache.get(change); - if (cached !== undefined) - return cached; - const version = this.#shallowStartVersion.length() === 0 - ? new Map() - : new Map(this.#shallowStartVersion - ._codecEntriesUnsorted() - .map(({ peer, counter }) => [peer, counter])); - for (const dependency of change.dependencies) { - const dependencyRecord = this.#recordContaining(dependency); - if (dependencyRecord !== undefined && dependencyRecord.change !== change) { - for (const [peer, counter] of this.#dependencyVersion(dependencyRecord.change)) { - version.set(peer, Math.max(version.get(peer) ?? 0, counter)); - } - } - version.set(dependency.peer, Math.max(version.get(dependency.peer) ?? 0, dependency.counter + 1)); - } - this.#dependencyVersionCache.set(change, version); - return version; - } - #frontiersForVersion(version) { - const candidates = new Map(); - const candidateByPeer = new Map(); - const known = this.#historyVersion(); - for (const { peer, counter } of version._codecEntriesUnsorted()) { - const end = Math.min(counter, known.get(peer) ?? 0); - if (end === 0) - continue; - const last = { peer, counter: end - 1 }; - if (this.#recordContaining(last) !== undefined) { - candidates.set(idKey(last), last); - candidateByPeer.set(peer, last); - } - } - for (const frontier of [...candidates.values()]) { - const causalVersion = this.#causalVersionAt([frontier]); - for (const [peer, counter] of causalVersion) { - const candidate = candidateByPeer.get(peer); - if (candidate !== undefined && - !idsEqual(candidate, frontier) && - candidate.counter < counter) { - candidates.delete(idKey(candidate)); - } - } - } - return [...candidates.values()].sort(compareIds); - } - #frontiersCodec() { - const candidates = new Map(); - if (this.#checkoutVersion !== undefined) { - for (const frontier of this.#frontiersForVersion(this.#checkoutVersion)) { - candidates.set(idKey(frontier), frontier); - } - } - else if (this.#deferredSnapshotHistory !== undefined) { - for (const frontier of this.#deferredSnapshotHistory.frontiers) { - candidates.set(idKey(frontier), frontier); - } - } - else { - for (const [key, frontier] of this.#historyFrontiers) { - candidates.set(key, frontier); - } - } - if (this.#pending !== undefined && this.#pending.operations.length > 0) { - for (const dependency of this.#pending.dependencies) - candidates.delete(idKey(dependency)); - const last = { - peer: this.#peer, - counter: this.#pending.id.counter + this.#pending.operationLength - 1, - }; - candidates.set(idKey(last), last); - } - return [...candidates.values()].sort(compareIds); - } - #lamportAt(id) { - const record = this.#recordContaining(id); - return record === undefined - ? 0 - : record.change.lamport + (id.counter - record.change.id.counter); - } - #encodeUpdates(records) { - const blocks = records.map((record) => encodeSingleChangeBlock(record.change, record.keys)); - return encodeDocument(EncodeMode.FastUpdates, encodeFastUpdatesBody(blocks)); - } - #encodeSnapshot() { - if (this.isShallow()) { - return this.#encodeShallowSnapshot(this.shallowSinceFrontiers()); - } - const historyEntries = this.#sortedHistory().map((record) => ({ - key: encodeChangeBlockKey(record.change.id), - value: encodeSingleChangeBlock(record.change, record.keys), - })); - historyEntries.push({ - key: VERSION_KEY, - value: encodePostcardVersionVector(this.version().codecEntries()), - }); - historyEntries.push({ - key: FRONTIERS_KEY, - value: encodePostcardFrontiers(this.#frontiersCodec()), - }); - const body = encodeFastSnapshotBody({ - oplog: encodeSstable(historyEntries, { compression: "auto" }), - state: encodeStateSnapshotStore(this.#buildStateStore(), { compression: "auto" }), - shallowRootState: new Uint8Array(), - }); - return encodeDocument(EncodeMode.FastSnapshot, body); - } - #encodeShallowSnapshot(requestedFrontiers) { - const startFrontiers = this.#calculateShallowStart(requestedFrontiers); - const rootVersion = this.#causalVersionForKnownFrontiers(startFrontiers); - const startVersion = rootVersion.clone(); - for (const frontier of startFrontiers) { - startVersion.set(frontier.peer, frontier.counter); - } - const latestVersion = this.#historyVersion(); - const latestFrontiers = this.#frontiersForVersion(latestVersion); - const restoreVersion = this.version(); - this.#rebuildFromHistory(rootVersion); - const rootStore = this.#buildStateStore(startFrontiers); - this.#rebuildFromHistory(latestVersion); - const latestStore = this.#buildStateStore(); - if (restoreVersion.compare(latestVersion) !== 0) { - this.#rebuildFromHistory(restoreVersion); - } - const historyEntries = this.#recordsInVersionRange(startVersion, latestVersion).map((record) => ({ - key: encodeChangeBlockKey(record.change.id), - value: encodeSingleChangeBlock(record.change, record.keys), - })); - historyEntries.push({ - key: VERSION_KEY, - value: encodePostcardVersionVector(latestVersion.codecEntries()), - }, { - key: FRONTIERS_KEY, - value: encodePostcardFrontiers(latestFrontiers), - }, { - key: START_VERSION_KEY, - value: encodePostcardVersionVector(startVersion.codecEntries()), - }, { - key: START_FRONTIERS_KEY, - value: encodePostcardFrontiers(startFrontiers), - }); - return encodeDocument(EncodeMode.FastSnapshot, encodeFastSnapshotBody({ - oplog: encodeSstable(historyEntries, { compression: "auto" }), - state: encodeStateSnapshotStore(latestStore, { compression: "auto" }), - shallowRootState: encodeStateSnapshotStore(rootStore, { - compression: "auto", - }), - })); - } - #calculateShallowStart(requested) { - const parsed = requested.map(parseOpId); - for (const frontier of requested) { - if (this.#recordContaining(parseOpId(frontier)) === undefined) { - throw new RangeError(`shallow snapshot frontier ${frontier.counter}@${frontier.peer} is unknown`); - } - } - let start = parsed; - if (start.length > 1) { - const versions = start.map((frontier) => this.#causalVersionAt([frontier])); - const peers = new Set(versions.flatMap((version) => [...version.keys()])); - const common = new VersionVector(); - for (const peer of peers) { - const counter = Math.min(...versions.map((version) => version.get(peer) ?? 0)); - if (counter > 0) - common.set(peer, counter); - } - const commonFrontiers = this.#frontiersForVersion(common); - start = commonFrontiers.length === 1 ? commonFrontiers : []; - } - if (start.length === 1) { - const operation = this.#operationAt(start[0]); - if (operation?.content.type === "text-mark") { - start = [{ peer: start[0].peer, counter: start[0].counter + 1 }]; - } - } - const candidateVersion = this.#causalVersionForKnownFrontiers(start); - if (this.isShallow() && - !versionIncludes(candidateVersion, this.#shallowRootVersion)) { - return this.#shallowRootFrontiers.map((id) => ({ ...id })); - } - return start; - } - #causalVersionForKnownFrontiers(frontiers) { - const version = new VersionVector(); - for (const [peer, counter] of this.#causalVersionAt(frontiers)) { - version.set(peer, counter); - } - return version; - } - #operationAt(id) { - const record = this.#recordContaining(id); - if (record === undefined) - return undefined; - let low = 0; - let high = record.change.operations.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (record.change.operations[middle].counter <= id.counter) - low = middle + 1; - else - high = middle; - } - const operation = low > 0 ? record.change.operations[low - 1] : undefined; - return operation !== undefined && id.counter < operation.counter + operation.length - ? operation - : undefined; - } - #buildStateStore(frontiers) { - const containers = []; - for (const container of this.#containers.values()) { - const id = container._codecId; - if (id === undefined) - continue; - const parent = container.parent()?._codecId; - containers.push({ - id, - wrapper: { - containerType: id.containerType, - depth: BigInt(containerDepth(container)), - parent, - state: this.#containerState(container), - }, - }); - } - return containers.length === 0 && frontiers === undefined - ? { kind: "empty" } - : { kind: "sstable", frontiers, containers }; - } - #containerState(container) { - if (container instanceof LoroMap) { - const peers = []; - const peerIndices = new Map(); - const peerIndex = (peer) => { - let index = peerIndices.get(peer); - if (index === undefined) { - index = peers.length; - peers.push(peer); - peerIndices.set(peer, index); - } - return BigInt(index); - }; - const values = []; - const deletedKeys = []; - const metadata = []; - for (const [key, record] of container._entries) { - if (record.deleted) - deletedKeys.push(key); - else - values.push([key, this.#encodeSnapshotValue(record.rawValue ?? record.value)]); - metadata.push({ - key, - peerIndex: peerIndex(record.writer.peer), - lamport: BigInt(record.writer.lamport), - }); - } - return { kind: CodecContainerType.Map, values, deletedKeys, peers, metadata }; - } - if (container instanceof LoroText) { - const peers = []; - const peerIndices = new Map(); - const peerIndex = (peer) => { - let index = peerIndices.get(peer); - if (index === undefined) { - index = peers.length; - peers.push(peer); - peerIndices.set(peer, index); - } - return BigInt(index); - }; - const keys = []; - const keyIndices = new Map(); - const keyIndex = (key) => { - let index = keyIndices.get(key); - if (index === undefined) { - index = keys.length; - keys.push(key); - keyIndices.set(key, index); - } - return index; - }; - const spans = []; - const marks = []; - const metasAt = container._attributeMetasResolver(); - let active = new Map(); - const updateActiveStyles = (current) => { - for (const [key, meta] of active) { - if (current.get(key) === meta) - continue; - spans.push({ - peerIndex: peerIndex(meta.startId.peer), - counter: meta.startId.counter + 1, - lamportSub: meta.lamport - meta.startId.counter, - length: -1, - }); - } - for (const [key, meta] of current) { - if (active.get(key) === meta) - continue; - spans.push({ - peerIndex: peerIndex(meta.startId.peer), - counter: meta.startId.counter, - lamportSub: meta.lamport - meta.startId.counter, - length: 0, - }); - marks.push({ - keyIndex: keyIndex(key), - value: this.#encodeSnapshotValue(meta.value), - info: meta.info, - }); - } - active = current; - }; - const textChunks = []; - let textChunk = ""; - let textChunkLength = 0; - const appendText = (text, scalarLength) => { - if (textChunkLength > 0 && textChunkLength + scalarLength > 1024) { - textChunks.push(textChunk); - textChunk = ""; - textChunkLength = 0; - } - textChunk += text; - textChunkLength += scalarLength; - if (textChunkLength === 1024) { - textChunks.push(textChunk); - textChunk = ""; - textChunkLength = 0; - } - }; - let cachedPeer; - let cachedPeerIndex = 0n; - const appendId = (peer, counter, lamport) => { - if (cachedPeer !== peer) { - cachedPeer = peer; - cachedPeerIndex = peerIndex(peer); - } - const lamportSub = lamport - counter; - const previous = spans[spans.length - 1]; - if (previous !== undefined && - previous.length > 0 && - previous.peerIndex === cachedPeerIndex && - previous.counter + previous.length === counter && - previous.lamportSub === lamportSub) { - previous.length += 1; - } - else { - spans.push({ - peerIndex: cachedPeerIndex, - counter, - lamportSub, - length: 1, - }); - } - }; - if (container._styleIndex.isEmpty) { - container._forEachVisibleSnapshotData(appendText, appendId); - } - else { - container._sequence.forEachVisible((element) => { - updateActiveStyles(metasAt(element)); - appendId(element.id.peer, element.id.counter, element.lamport); - appendText(element.value, 1); - }); - } - updateActiveStyles(new Map()); - if (textChunkLength > 0) - textChunks.push(textChunk); - return { - kind: CodecContainerType.Text, - text: textChunks.join(""), - peers, - spans, - keys, - marks, - }; - } - if (container instanceof LoroTree) { - // Rust's fast-tree decoder historically accepted the canonical snapshot - // order: live roots/subtrees first, then deleted roots/subtrees, with each - // sibling group ordered by fractional position and last-move writer. - // Preserve that order so snapshots remain byte-level interoperable and so - // parent-local ordered insertion never sees an insertion-order permutation. - const records = container._recordsForSnapshot(); - const peers = []; - const peerIndices = new Map(); - const peerIndex = (peer) => { - let index = peerIndices.get(peer); - if (index === undefined) { - index = peers.length; - peers.push(peer); - peerIndices.set(peer, index); - } - return BigInt(index); - }; - const positions = []; - const positionIndices = new Map(); - const recordIndices = new Map(records.map((record, index) => [idKey(record.id), index])); - return { - kind: CodecContainerType.Tree, - peers, - nodes: records.map((record) => { - const positionKey = bytesToHex(record.position); - let positionIndex = positionIndices.get(positionKey); - if (positionIndex === undefined) { - positionIndex = positions.length; - positions.push(record.position.slice()); - positionIndices.set(positionKey, positionIndex); - } - const parentIndex = record.deleted - ? 1n - : record.parent === undefined - ? 0n - : BigInt((recordIndices.get(idKey(record.parent)) ?? -2) + 2); - return { - peerIndex: peerIndex(record.id.peer), - counter: record.id.counter, - parentIndexPlusTwo: parentIndex, - lastSetPeerIndex: peerIndex(record.writer.peer), - lastSetCounter: record.lastMoveId.counter, - lastSetLamportSub: record.writer.lamport - record.lastMoveId.counter, - fractionalIndexIndex: positionIndex, - }; - }), - positions, - reserved: new Uint8Array(), - }; - } - if (container instanceof LoroCounter) { - float64BitsScratch.setFloat64(0, container.value, true); - return { - kind: CodecContainerType.Counter, - bits: float64BitsScratch.getBigUint64(0, true), - }; - } - if (!(container instanceof LoroList)) { - throw new TypeError(`unsupported container kind ${container.kind()}`); - } - const visible = container._visibleElements(); - const peers = []; - const peerIndices = new Map(); - const peerIndex = (peer) => { - let index = peerIndices.get(peer); - if (index === undefined) { - index = peers.length; - peers.push(peer); - peerIndices.set(peer, index); - } - return BigInt(index); - }; - if (container instanceof LoroMovableList) { - // The state-snapshot encoder only reads item fields, so every slot can - // reference the same frozen item instead of allocating one per element. - const items = [SHARED_MOVABLE_LIST_STATE_ITEM]; - for (let index = 0; index < visible.length; index += 1) { - items.push(SHARED_MOVABLE_LIST_STATE_ITEM); - } - return { - kind: CodecContainerType.MovableList, - values: visible.map((element) => this.#encodeSnapshotValue(element.value)), - peers, - items, - listItemIds: visible.map((element) => ({ - peerIndex: peerIndex(element.id.peer), - counter: element.id.counter, - lamportSub: element.lamport - element.id.counter, - })), - elementIds: [], - lastSetIds: [], - }; - } - return { - kind: CodecContainerType.List, - values: visible.map((element) => this.#encodeSnapshotValue(element.value)), - peers, - ids: visible.map((element) => ({ - peerIndex: peerIndex(element.id.peer), - counter: element.id.counter, - lamportSub: element.lamport - element.id.counter, - })), - }; - } - #encodeSnapshotValue(value) { - if (isContainer(value)) { - if (value._codecId === undefined) - throw new Error("detached child in attached state"); - return { type: "container", value: value._codecId }; - } - if (value === null) - return { type: "null" }; - if (typeof value === "boolean") - return { type: "bool", value }; - if (typeof value === "number") { - return Number.isSafeInteger(value) - ? { type: "i64", value: BigInt(value) } - : { type: "double", value }; - } - if (typeof value === "string") - return { type: "string", value }; - if (value instanceof Uint8Array) - return { type: "binary", value: value.slice() }; - if (Array.isArray(value)) - return { - type: "list", - value: value.map((item) => this.#encodeSnapshotValue(item)), - }; - return { - type: "map", - value: Object.entries(value).map(([key, item]) => [ - key, - this.#encodeSnapshotValue(item), - ]), - }; - } - #hydrateState(store, textSnapshotIds = validateStateSnapshotTextIds(store)) { - if (store.kind !== "sstable") - return; - let remainingTextCounterSlots = 10000000; - for (const { id } of store.containers) - this.#getOrCreateContainer(id); - for (const { id, wrapper } of store.containers) { - const container = this.#getOrCreateContainer(id); - const parent = wrapper.parent === undefined - ? undefined - : this.#getOrCreateContainer(wrapper.parent); - container._attach(this, id, parent); - container._reset(); - } - for (const { id, wrapper } of store.containers) { - const container = this.#getOrCreateContainer(id); - const state = wrapper.state; - if (container instanceof LoroMap && state.kind === CodecContainerType.Map) { - const metadata = new Map(state.metadata.map((item) => [item.key, item])); - for (const [key, value] of state.values) { - const item = metadata.get(key); - const rawValue = this.#decodeSnapshotValue(value, container); - container._applyValue(key, this.#materializeMapValue(container, key, rawValue), { - peer: state.peers[Number(item.peerIndex)], - lamport: Number(item.lamport), - }, rawValue); - } - for (const key of state.deletedKeys) { - const item = metadata.get(key); - container._applyDelete(key, { - peer: state.peers[Number(item.peerIndex)], - lamport: Number(item.lamport), - }); - } - } - else if (container instanceof LoroText && - state.kind === CodecContainerType.Text) { - const snapshotIds = textSnapshotIds.get(state); - for (const { peer, end, visible } of snapshotIds.reservations) { - if (end > 0 && end <= visible * 4 + 4096 && end <= remainingTextCounterSlots) { - container._sequence.reservePeerCounters(peer, end); - remainingTextCounterSlots -= end; - } - } - container._beginValidatedSnapshotLoad(); - const styleRuns = []; - const stylesById = new Map(); - const active = new Map(); - let textUtf16Offset = 0; - let chunkTextStart = 0; - const chunkPeers = []; - const chunkCounters = []; - const chunkLamports = []; - const flushChunk = () => { - if (chunkPeers.length === 0) - return; - container._insertValidatedSnapshotChunk(state.text.slice(chunkTextStart, textUtf16Offset), chunkPeers, chunkCounters, chunkLamports); - chunkPeers.length = 0; - chunkCounters.length = 0; - chunkLamports.length = 0; - chunkTextStart = textUtf16Offset; - }; - let markIndex = 0; - for (const span of state.spans) { - const peer = state.peers[Number(span.peerIndex)]; - if (span.length === 0) { - const mark = state.marks[markIndex++]; - const key = state.keys[mark.keyIndex]; - const value = this.#decodeSnapshotValue(mark.value); - const meta = { - startId: { peer, counter: span.counter }, - lamport: span.counter + span.lamportSub, - info: mark.info, - value, - }; - stylesById.set(idKey(meta.startId), { key, meta }); - const stack = active.get(key) ?? []; - stack.push(meta); - active.set(key, stack); - continue; - } - if (span.length === -1) { - const style = stylesById.get(idKey({ peer, counter: span.counter - 1 })); - if (style !== undefined) { - const stack = active.get(style.key); - if (stack !== undefined) { - const index = stack.lastIndexOf(style.meta); - if (index >= 0) - stack.splice(index, 1); - if (stack.length === 0) - active.delete(style.key); - } - } - continue; - } - if (span.length < -1) - continue; - for (const [key, stack] of active) { - const meta = stack.at(-1); - if (meta !== undefined) { - styleRuns.push({ - run: { start: { peer, counter: span.counter }, length: span.length }, - key, - meta, - }); - } - } - for (let offset = 0; offset < span.length; offset += 1) { - // Inline advanceUnicodeScalars(state.text, textUtf16Offset, 1). - if (textUtf16Offset >= state.text.length) { - throw new Error("text snapshot span exceeds the decoded text"); - } - const first = state.text.charCodeAt(textUtf16Offset); - textUtf16Offset += 1; - if (first >= 0xd800 && - first <= 0xdbff && - textUtf16Offset < state.text.length && - state.text.charCodeAt(textUtf16Offset) >= 0xdc00 && - state.text.charCodeAt(textUtf16Offset) <= 0xdfff) { - textUtf16Offset += 1; - } - chunkPeers.push(peer); - chunkCounters.push(span.counter + offset); - chunkLamports.push(span.counter + offset + span.lamportSub); - if (chunkPeers.length === 32) - flushChunk(); - } - } - flushChunk(); - if (textUtf16Offset !== state.text.length) { - throw new Error("text snapshot spans do not consume the decoded text"); - } - container._endValidatedSnapshotLoad(); - for (const { run, key, meta } of styleRuns) { - container._styleIndex.add([run], key, meta); - } - container._attributeHistoryComplete = false; - } - else if (container instanceof LoroTree && - state.kind === CodecContainerType.Tree) { - const records = []; - for (let index = 0; index < state.nodes.length; index += 1) { - const node = state.nodes[index]; - const nodeId = { - peer: state.peers[Number(node.peerIndex)], - counter: node.counter, - }; - const dataId = { - kind: "normal", - ...nodeId, - containerType: CodecContainerType.Map, - }; - const record = { - id: nodeId, - parent: undefined, - position: state.positions[node.fractionalIndexIndex].slice(), - deleted: node.parentIndexPlusTwo === 1n, - writer: { - peer: state.peers[Number(node.lastSetPeerIndex)], - lamport: node.lastSetCounter + node.lastSetLamportSub, - }, - lastMoveId: { - peer: state.peers[Number(node.lastSetPeerIndex)], - counter: node.lastSetCounter, - }, - data: this.#getOrCreateContainer(dataId, container), - }; - records.push(record); - } - for (let index = 0; index < state.nodes.length; index += 1) { - const parent = state.nodes[index].parentIndexPlusTwo; - if (parent >= 2n) - records[index].parent = records[Number(parent - 2n)].id; - } - for (const record of records) - container._setRecord(record); - } - else if (container instanceof LoroCounter && - state.kind === CodecContainerType.Counter) { - float64BitsScratch.setBigUint64(0, state.bits, true); - container._value = float64BitsScratch.getFloat64(0, true); - } - else if (container instanceof LoroMovableList && - state.kind === CodecContainerType.MovableList) { - const ids = state.listItemIds.slice(0, state.values.length); - container._insertVisible(0, state.values.map((value) => this.#decodeSnapshotValue(value, container)), ids.map((item) => ({ - peer: state.peers[Number(item.peerIndex)], - counter: item.counter, - })), ids.map((item) => item.counter + item.lamportSub)); - container._valueHistoryComplete = false; - container._moveHistoryComplete = false; - } - else if (container instanceof LoroList && - state.kind === CodecContainerType.List) { - container._insertVisible(0, state.values.map((value) => this.#decodeSnapshotValue(value, container)), state.ids.map((item) => ({ - peer: state.peers[Number(item.peerIndex)], - counter: item.counter, - })), state.ids.map((item) => item.counter + item.lamportSub)); - } - } - } - #decodeSnapshotValue(value, parent) { - switch (value.type) { - case "null": - return null; - case "bool": - return value.value; - case "double": - return value.value; - case "i64": - return Number(value.value); - case "string": - return value.value; - case "binary": - return value.value.slice(); - case "list": - return value.value.map((item) => this.#decodeSnapshotValue(item)); - case "map": - return Object.fromEntries(value.value.map(([key, item]) => [key, this.#decodeSnapshotValue(item)])); - case "container": - return this.#getOrCreateContainer(value.value, parent); - } - } - #materializeMapValue(parent, key, rawValue) { - const parentId = parent._codecId; - if (parentId === undefined) - return rawValue; - const childType = parseMergeableMarker(parentId, key, rawValue); - if (childType === undefined) - return rawValue; - return this.#getOrCreateContainer(newMergeableContainerId(parentId, key, childType), parent); - } - #captureEventValues(records) { - const values = new Map(); - for (const { change } of records) { - for (const operation of change.operations) { - const id = this.#containerKey(operation.container); - if (values.has(id)) - continue; - const container = this.#containers.get(id); - values.set(id, container === undefined ? undefined : containerEventValue(container)); - } - } - return values; - } - #recordConcurrentTreeSubjects(recording, added, beforeVersion) { - const incomingVersion = new VersionVector(); - for (const { change } of added) { - const length = changeLength(change); - if (length === 0) - continue; - const frontier = { - peer: change.id.peer, - counter: change.id.counter + length - 1, - }; - for (const [peer, counter] of this.#causalVersionAt([frontier])) { - if (counter > (incomingVersion.get(peer) ?? 0)) { - incomingVersion.set(peer, counter); - } - } - } - // Rust calculates a concurrent import diff by retreating to the common - // ancestor and replaying both branches. Tree operations from the local-only - // branch can therefore produce visible reindex moves even though those - // operations are not present in the imported update. Capture those subjects - // so the incremental TS merge reports the same net transition. - for (const { change } of this.#recordsInVersionRange(incomingVersion, beforeVersion)) { - for (const operation of change.operations) { - const content = operation.content; - if (content.type !== "tree-create" && - content.type !== "tree-move" && - content.type !== "tree-delete") { - continue; - } - const container = this.#containers.get(this.#containerKey(operation.container)); - if (!(container instanceof LoroTree)) - continue; - const state = this.#treeEventState(recording, container); - const nodeId = formatTreeId(content.subject); - if (state.originals.has(nodeId)) - continue; - const record = container._nodes.get(nodeId); - state.originals.set(nodeId, record === undefined || container._isRecordDeleted(record) - ? undefined - : this.#treeEventNode(container, record)); - } - } - } - #captureContainerEventValues(ids) { - const values = new Map(); - for (const id of ids) { - const container = this.#containers.get(id); - values.set(id, container === undefined ? undefined : containerEventValue(container)); - } - return values; - } - #captureMapKeys(ids) { - const keys = new Map(); - for (const id of ids) { - const container = this.#containers.get(id); - if (container instanceof LoroMap) - keys.set(id, new Set(container._entries.keys())); - } - return keys; - } - #emit(by, origin, from, to, changed, beforeValues = new Map(), preparedDiffs = new Map()) { - if (changed.size === 0 || !this.#hasEventSubscribers()) - return; - const expandedChanged = new Set(changed); - const revived = new Set(); - const calculatedDiffs = new Map(preparedDiffs); - for (const id of changed) { - const container = this.#containers.get(id); - if (!(container instanceof LoroTree) || this._isContainerDeleted(container)) { - continue; - } - const diff = calculatedDiffs.get(id) ?? containerDiff(container, beforeValues.get(id)); - calculatedDiffs.set(id, diff); - if (diff.type !== "tree") - continue; - for (const item of diff.diff) { - if (item.action !== "create") - continue; - const record = container._nodes.get(item.target); - if (record === undefined || container._isRecordDeleted(record)) - continue; - expandedChanged.add(record.data.id); - revived.add(record.data.id); - } - } - const events = []; - for (const id of expandedChanged) { - const container = this.#containers.get(id); - if (container === undefined || this._isContainerDeleted(container)) - continue; - const diff = calculatedDiffs.get(id) ?? - containerDiff(container, revived.has(id) ? undefined : beforeValues.get(id)); - if (isEmptyContainerDiff(diff)) - continue; - events.push({ target: id, diff, path: containerPath(container) }); - } - if (events.length === 0) - return; - const fromFormatted = from.map(formatOpId); - const toFormatted = to.map(formatOpId); - // Two explicit literals: the key set differs by `origin`, so each branch - // keeps one stable object shape and avoids CopyDataProperties. - const base = (origin === undefined - ? { by, events, from: fromFormatted, to: toFormatted } - : { by, origin, events, from: fromFormatted, to: toFormatted }); - for (const listener of this.#subscribers) - listener(base); - const relevantByTarget = new Map(); - for (const event of events) { - let current = this.#containers.get(event.target); - while (current !== undefined) { - const target = current.id; - if (this.#containerSubscribers.has(target)) { - const relevant = relevantByTarget.get(target); - if (relevant === undefined) - relevantByTarget.set(target, [event]); - else - relevant.push(event); - } - current = current.parent(); - } - } - for (const [target, relevant] of relevantByTarget) { - const listeners = this.#containerSubscribers.get(target); - if (listeners === undefined) - continue; - const batch = origin === undefined - ? { - by, - events: relevant, - from: fromFormatted, - to: toFormatted, - currentTarget: target, - } - : { - by, - origin, - events: relevant, - from: fromFormatted, - to: toFormatted, - currentTarget: target, - }; - for (const listener of listeners) - listener(batch); - } - } - #hasEventSubscribers() { - return this.#subscribers.size > 0 || this.#containerSubscribers.size > 0; - } -} -export class ChangeModifier { - #options; - constructor(options) { - this.#options = options; - } - free() { } - setMessage(message) { - this.#options.message = message; - return this; - } - setTimestamp(timestamp) { - if (!Number.isFinite(timestamp)) - throw new TypeError("timestamp must be finite"); - this.#options.timestamp = timestamp; - return this; - } -} -export class Loro extends LoroDoc { -} -export function callPendingEvents() { } -export function LORO_VERSION() { - return packageMetadata.version; -} -export function encodeFrontiers(frontiers) { - return encodePostcardFrontiers(frontiers.map(parseOpId)); -} -export function decodeFrontiers(bytes) { - return decodePostcardFrontiers(bytes).map(formatOpId); -} -export function setDebug() { } -export function decodeImportBlobMeta(blob, checkChecksum = true) { - const parsed = decodeDocument(blob, { checkChecksum }); - if (parsed.mode === EncodeMode.FastUpdates) { - const records = decodeFastUpdatesBody(parsed.body).flatMap(decodeHistoryRecordBlock); - const startVersion = new VersionVector(); - const endVersion = new VersionVector(); - for (const { change } of records) { - const currentStart = startVersion.get(change.id.peer); - if (currentStart === undefined || change.id.counter < currentStart) { - startVersion.set(change.id.peer, change.id.counter); - } - const end = change.id.counter + changeLength(change); - if (end > (endVersion.get(change.id.peer) ?? 0)) { - endVersion.set(change.id.peer, end); - } - } - const startFrontiers = new Map(); - for (const { change } of records) { - for (const dependency of change.dependencies) { - const start = startVersion.get(dependency.peer); - if ((start !== undefined && start > dependency.counter) || - (start === undefined && endVersion.get(dependency.peer) === undefined)) { - startFrontiers.set(idKey(dependency), dependency); - } - } - } - return { - partialStartVersionVector: startVersion, - partialEndVersionVector: endVersion, - startFrontiers: [...startFrontiers.values()].map(formatOpId), - startTimestamp: Number(records[0]?.change.timestamp ?? 0n), - endTimestamp: Number(records.at(-1)?.change.timestamp ?? 0n), - mode: "update", - changeNum: records.length, - }; - } - const snapshot = decodeFastSnapshotBody(parsed.body); - const records = []; - let startVersion = new VersionVector(); - let endVersion = new VersionVector(); - let startFrontiers = []; - let endFrontiers = []; - for (const entry of decodeSstable(snapshot.oplog)) { - if (bytesEqual(entry.key, VERSION_KEY)) { - endVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); - } - else if (bytesEqual(entry.key, FRONTIERS_KEY)) { - endFrontiers = decodePostcardFrontiers(entry.value); - } - else if (bytesEqual(entry.key, START_VERSION_KEY)) { - startVersion = versionVectorFromCodec(decodePostcardVersionVector(entry.value)); - } - else if (bytesEqual(entry.key, START_FRONTIERS_KEY)) { - startFrontiers = decodePostcardFrontiers(entry.value); - } - else if (entry.key.length === 12) { - records.push(...decodeHistoryRecordBlock(entry.value)); - } - } - if (endVersion.length() === 0) { - for (const { change } of records) { - const end = change.id.counter + changeLength(change); - if (end > (endVersion.get(change.id.peer) ?? 0)) - endVersion.set(change.id.peer, end); - } - } - const timestampAt = (frontiers) => { - const targetByPeer = new Map(frontiers.map((id) => [id.peer, id])); - let timestamp = 0; - for (const { change } of records) { - const frontier = targetByPeer.get(change.id.peer); - if (frontier === undefined) - continue; - const end = change.id.counter + changeLength(change); - if (frontier.counter >= change.id.counter && frontier.counter < end) { - timestamp = Math.max(timestamp, Number(change.timestamp)); - } - } - return timestamp; - }; - let latestRecord; - for (const record of records) { - if (latestRecord === undefined || compareHistoryRecords(latestRecord, record) < 0) { - latestRecord = record; - } - } - return { - partialStartVersionVector: startVersion, - partialEndVersionVector: endVersion, - startFrontiers: startFrontiers.map(formatOpId), - startTimestamp: timestampAt(startFrontiers), - endTimestamp: timestampAt(endFrontiers) || Number(latestRecord?.change.timestamp ?? 0n), - mode: snapshot.shallowRootState.length > 0 ? "shallow-snapshot" : "snapshot", - changeNum: records.length, - }; -} -function decodeHistoryRecordBlock(bytes) { - const block = decodeChangeBlock(bytes); - return block.changes.map((change) => ({ - change, - keys: block.keys, - keyIndices: undefined, - })); -} -function versionVectorFromCodec(ids) { - return new VersionVector(new Map(ids.map((id) => [peerIdToString(id.peer), id.counter]))); -} -function diffContainerId(value) { - if (isContainer(value)) - return value.id; - if (typeof value !== "string" || !value.startsWith("🦜:")) - return undefined; - const id = value.slice("🦜:".length); - return isContainerId(id) ? id : undefined; -} -function ensureMergeableChild(parent, key, type) { - switch (type) { - case "Map": - return parent.ensureMergeableMap(key); - case "List": - return parent.ensureMergeableList(key); - case "MovableList": - return parent.ensureMergeableMovableList(key); - case "Text": - return parent.ensureMergeableText(key); - case "Tree": - return parent.ensureMergeableTree(key); - case "Counter": - return parent.ensureMergeableCounter(key); - } -} -function diffKindMismatch(container, diffType) { - return new TypeError(`cannot apply a ${diffType} diff to a ${container.kind()} container`); -} -function createContainer(type) { - switch (type) { - case "Map": - return new LoroMap(); - case "List": - return new LoroList(); - case "Text": - return new LoroText(); - case "Tree": - return new LoroTree(); - case "MovableList": - return new LoroMovableList(); - case "Counter": - return new LoroCounter(); - } -} -function registerKey(pending, key) { - const existing = pending.keyIndices.get(key); - if (existing !== undefined) - return existing; - const index = pending.keys.length; - pending.keys.push(key); - pending.keyIndices.set(key, index); - return index; -} -function changeKey(id) { - return `${id.peer}:${id.counter}`; -} -function idKey(id) { - return `${id.peer}:${id.counter}`; -} -function frontierSetsEqual(left, right) { - if (left.length !== right.length) - return false; - const rightIds = new Set(right.map((id) => idKey(parseOpId(id)))); - return left.every((id) => rightIds.has(idKey(parseOpId(id)))); -} -const changeLengthCache = new WeakMap(); -function changeLength(change) { - const cached = changeLengthCache.get(change); - if (cached !== undefined) - return cached; - const length = change.operations.reduce((sum, operation) => sum + operation.length, 0); - changeLengthCache.set(change, length); - return length; -} -function validateStateSnapshotTextIds(store) { - const textSnapshotIds = new Map(); - if (store.kind !== "sstable") - return textSnapshotIds; - for (const { wrapper } of store.containers) { - if (wrapper.state.kind === CodecContainerType.Text) { - textSnapshotIds.set(wrapper.state, validateTextSnapshotIdRanges(wrapper.state.peers, wrapper.state.spans)); - } - } - return textSnapshotIds; -} -function validateTextSnapshotIdRanges(peers, spans) { - const statsByPeer = new Map(); - for (const span of spans) { - if (span.length <= 0) - continue; - const peerIndex = Number(span.peerIndex); - const end = span.counter + span.length; - if (!Number.isSafeInteger(peerIndex) || - peerIndex < 0 || - peerIndex >= peers.length || - !Number.isSafeInteger(span.counter) || - span.counter < 0 || - !Number.isSafeInteger(end) || - end <= span.counter || - end > 2147483647) { - throw new Error("text snapshot contains an invalid ID range"); - } - const peer = peers[peerIndex]; - let stats = statsByPeer.get(peer); - if (stats === undefined) { - stats = { peer, count: 0, end: 0, visible: 0, offset: 0 }; - statsByPeer.set(peer, stats); - } - stats.count += 1; - stats.visible += span.length; - if (!Number.isSafeInteger(stats.visible)) { - throw new Error("text snapshot contains too many IDs for one peer"); - } - if (end > stats.end) - stats.end = end; - } - for (const stats of statsByPeer.values()) { - stats.ranges = new BigUint64Array(stats.count); - } - for (const span of spans) { - if (span.length <= 0) - continue; - const peerIndex = Number(span.peerIndex); - const end = span.counter + span.length; - const stats = statsByPeer.get(peers[peerIndex]); - stats.ranges[stats.offset] = (BigInt(span.counter) << 32n) | BigInt(end); - stats.offset += 1; - } - for (const stats of statsByPeer.values()) { - const peerRanges = stats.ranges; - peerRanges.sort(); - let previousEnd = -1; - for (const packed of peerRanges) { - const start = Number(packed >> 32n); - const end = Number(packed & 0xffffffffn); - if (start < previousEnd) { - throw new Error("text snapshot contains duplicate sequence IDs"); - } - previousEnd = end; - } - } - return { - reservations: [...statsByPeer.values()].map(({ peer, end, visible }) => ({ - peer, - end, - visible, - })), - }; -} -function changedContainerIds(records) { - return new Set(records.flatMap(({ change }) => change.operations.map((operation) => formatContainerId(operation.container)))); -} -function longestIncreasingSubsequenceIndices(values) { - if (values.length === 0) - return new Set(); - const tails = []; - const previous = new Int32Array(values.length); - previous.fill(-1); - for (let index = 0; index < values.length; index += 1) { - let low = 0; - let high = tails.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (values[tails[middle]] < values[index]) - low = middle + 1; - else - high = middle; - } - if (low > 0) - previous[index] = tails[low - 1]; - tails[low] = index; - } - const indices = new Set(); - let index = tails.at(-1); - while (index >= 0) { - indices.add(index); - index = previous[index]; - } - return indices; -} -function movableMoveTransitionMode(retreat, forward, movePeers) { - const changedMoveContainers = new Set(); - for (const { change } of [...retreat, ...forward]) { - for (const operation of change.operations) { - if (operation.content.type === "movable-list-move") { - changedMoveContainers.add(formatContainerId(operation.container)); - } - } - } - if (changedMoveContainers.size === 0) - return "anchors"; - if (retreat.length > 0 && forward.length > 0) - return "replay"; - for (const containerId of changedMoveContainers) { - if ((movePeers.get(containerId)?.size ?? 0) > 1) - return "replay"; - } - return "anchors"; -} -function hasMaterializedSequenceInsertions(records, containers) { - for (const { change } of records) { - for (const operation of change.operations) { - const content = operation.content; - if (content.type !== "text-insert" && - content.type !== "list-insert" && - content.type !== "movable-list-insert") { - continue; - } - const container = containers.get(formatContainerId(operation.container)); - if ((container instanceof LoroText || container instanceof LoroList) && - container._sequence.findById({ - peer: change.id.peer, - counter: operation.counter, - }) !== undefined) { - return true; - } - } - } - return false; -} -function canMergeChanges(left, right, interval) { - return (right.id.peer === left.id.peer && - right.id.counter === left.id.counter + changeLength(left) && - right.dependencies.length === 1 && - right.dependencies[0].peer === left.id.peer && - right.timestamp - left.timestamp <= interval && - right.message === left.message); -} -function appendHistoryRecord(left, right, leftLength) { - let keyIndices = left.keyIndices; - let keys; - if (keyIndices === undefined) { - keys = [...left.keys]; - keyIndices = new Map(); - for (const [index, key] of keys.entries()) { - if (!keyIndices.has(key)) - keyIndices.set(key, index); - } - left.keys = keys; - left.keyIndices = keyIndices; - } - else { - keys = left.keys; - } - // A change with no keys of its own needs no remapping, so its operations - // array can be reused as-is; nothing downstream mutates it. - let appendedOperations; - if (right.keys.length === 0) { - appendedOperations = right.change.operations; - } - else { - const remappedKeyIndices = right.keys.map((key) => { - const existing = keyIndices.get(key); - if (existing !== undefined) - return existing; - const index = keys.length; - keys.push(key); - keyIndices.set(key, index); - return index; - }); - appendedOperations = right.change.operations.map((operation) => remapOperationKeys(operation, remappedKeyIndices)); - } - const mergedOperations = left.change.operations; - for (const operation of appendedOperations) - mergedOperations.push(operation); - const rightLength = changeLength(right.change); - changeLengthCache.set(left.change, leftLength + rightLength); - const appended = { - change: { - ...right.change, - timestamp: left.change.timestamp, - message: left.change.message, - operations: appendedOperations, - }, - keys, - // Deliberately not the merged map: it is shared with `left` and keeps - // growing, and this slice record never merges further. - keyIndices: undefined, - }; - changeLengthCache.set(appended.change, rightLength); - return appended; -} -function cloneHistoryRecord(record) { - return { - change: { ...record.change, operations: [...record.change.operations] }, - keys: record.keys, - keyIndices: undefined, - }; -} -function appendToTrailingListInsert(operations, operation) { - const content = operation.content; - if (content.type !== "list-insert" && content.type !== "movable-list-insert") { - return false; - } - const previousIndex = operations.length - 1; - const previous = operations[previousIndex]; - if (previous === undefined || - previous.content.type !== content.type || - !containerIdsEqual(previous.container, operation.container) || - previous.counter + previous.length !== operation.counter || - previous.content.position + previous.length !== content.position) { - return false; - } - const values = previous.content.values; - for (const value of content.values) - values.push(value); - operations[previousIndex] = { - ...previous, - length: previous.length + operation.length, - content: { ...previous.content, values }, - }; - return true; -} -function remapOperationKeys(operation, remap) { - const content = operation.content; - switch (content.type) { - case "map-insert": - case "text-mark": - case "movable-list-set": - return { - ...operation, - content: { ...content, value: remapLoroValueKeys(content.value, remap) }, - }; - case "list-insert": - case "movable-list-insert": - return { - ...operation, - content: { - ...content, - values: content.values.map((value) => remapLoroValueKeys(value, remap)), - }, - }; - case "future": - return { - ...operation, - content: { ...content, value: remapChangeValueKeys(content.value, remap) }, - }; - default: - return operation; - } -} -function remapChangeValueKeys(value, remap) { - switch (value.type) { - case "loro-value": - return { ...value, value: remapLoroValueKeys(value.value, remap) }; - case "mark-start": - return { - ...value, - keyIndex: remapKeyIndex(value.keyIndex, remap), - value: remapLoroValueKeys(value.value, remap), - }; - case "list-set": - return { ...value, value: remapLoroValueKeys(value.value, remap) }; - default: - return value; - } -} -function remapLoroValueKeys(value, remap) { - if (value.type === "list") { - return { - ...value, - value: value.value.map((item) => remapLoroValueKeys(item, remap)), - }; - } - if (value.type === "map") { - return { - ...value, - value: value.value.map(([keyIndex, item]) => [ - remapKeyIndex(keyIndex, remap), - remapLoroValueKeys(item, remap), - ]), - }; - } - return value; -} -function remapKeyIndex(keyIndex, remap) { - if (keyIndex < 0n || keyIndex > BigInt(Number.MAX_SAFE_INTEGER)) { - throw new Error("change value map key index is out of range"); - } - const mapped = remap[Number(keyIndex)]; - if (mapped === undefined) { - throw new Error("change value map key index is out of range"); - } - return BigInt(mapped); -} -const I64_MAX = 0x7fffffffffffffffn; -const I64_MIN = -0x8000000000000000n; -function numberToI64(value) { - if (Number.isNaN(value)) - return 0n; - if (value >= Number(I64_MAX)) - return I64_MAX; - if (value <= Number(I64_MIN)) - return I64_MIN; - return BigInt(Math.trunc(value)); -} -function maxBigInt(left, right) { - return left >= right ? left : right; -} -function sliceChange(change, from, to) { - const totalLength = changeLength(change); - if (!Number.isSafeInteger(from) || - !Number.isSafeInteger(to) || - from < 0 || - from >= to || - to > totalLength) { - throw new RangeError("change slice is out of range"); - } - const operations = []; - let offset = 0; - for (const operation of change.operations) { - const operationStart = offset; - const operationEnd = offset + operation.length; - offset = operationEnd; - if (operationEnd <= from) - continue; - if (operationStart >= to) - break; - const sliceFrom = Math.max(0, from - operationStart); - const sliceTo = Math.min(operation.length, to - operationStart); - if (sliceFrom === 0 && sliceTo === operation.length) { - operations.push(operation); - } - else { - operations.push(sliceOperation(operation, sliceFrom, sliceTo)); - } - } - return { - ...change, - id: { peer: change.id.peer, counter: change.id.counter + from }, - lamport: change.lamport + from, - dependencies: from === 0 - ? change.dependencies - : [{ peer: change.id.peer, counter: change.id.counter + from - 1 }], - operations, - }; -} -// Slice a string by Unicode scalar offsets without materializing per-scalar -// strings. Out-of-range offsets clamp like Array.from(value).slice would. -function sliceUnicodeScalars(value, from, to) { - let offset = 0; - let scalar = 0; - while (scalar < from && offset < value.length) { - const first = value.charCodeAt(offset); - offset += - first >= 0xd800 && - first <= 0xdbff && - offset + 1 < value.length && - value.charCodeAt(offset + 1) >= 0xdc00 && - value.charCodeAt(offset + 1) <= 0xdfff - ? 2 - : 1; - scalar += 1; - } - const start = offset; - while (scalar < to && offset < value.length) { - const first = value.charCodeAt(offset); - offset += - first >= 0xd800 && - first <= 0xdbff && - offset + 1 < value.length && - value.charCodeAt(offset + 1) >= 0xdc00 && - value.charCodeAt(offset + 1) <= 0xdfff - ? 2 - : 1; - scalar += 1; - } - return value.slice(start, offset); -} -function sliceOperation(operation, from, to) { - if (from < 0 || from >= to || to > operation.length) { - throw new RangeError("partial operation length is out of range"); - } - const length = to - from; - const content = operation.content; - switch (content.type) { - case "text-insert": - return { - ...operation, - counter: operation.counter + from, - length, - content: { - ...content, - position: content.position + from, - value: sliceUnicodeScalars(content.value, from, to), - }, - }; - case "list-insert": - case "movable-list-insert": - return { - ...operation, - counter: operation.counter + from, - length, - content: { - ...content, - position: content.position + from, - values: content.values.slice(from, to), - }, - }; - case "text-delete": - case "list-delete": - case "movable-list-delete": { - const signedLength = Number(content.length); - const positive = signedLength >= 0; - return { - ...operation, - counter: operation.counter + from, - length, - content: { - ...content, - position: positive ? content.position : content.position - from, - length: BigInt(positive ? length : -length), - startId: { - peer: content.startId.peer, - counter: content.startId.counter + (positive ? from : operation.length - to), - }, - }, - }; - } - default: - throw new Error(`cannot slice ${content.type} operation`); - } -} -function versionIncludes(version, required) { - return required - ._codecEntriesUnsorted() - .every(({ peer, counter }) => (version.get(peer) ?? 0) >= counter); -} -function historyVersionForRecords(records, base) { - const version = base?.clone() ?? new VersionVector(); - for (const { change } of records) { - const end = change.id.counter + changeLength(change); - if (end > (version.get(change.id.peer) ?? 0)) { - version.set(change.id.peer, end); - } - } - return version; -} -function versionDistance(start, end) { - let distance = 0; - for (const { peer, counter } of end._codecEntriesUnsorted()) { - distance += Math.max(0, counter - (start.get(peer) ?? 0)); - } - return distance; -} -function mergeStateSnapshotStores(root, overlay) { - if (root.kind !== "sstable") - return overlay; - if (overlay.kind !== "sstable") { - return { kind: "sstable", frontiers: undefined, containers: root.containers }; - } - const containers = new Map(root.containers.map((entry) => [formatContainerId(entry.id), entry])); - for (const entry of overlay.containers) { - containers.set(formatContainerId(entry.id), entry); - } - return { - kind: "sstable", - frontiers: undefined, - containers: [...containers.values()], - }; -} -function compareIds(left, right) { - return left.peer < right.peer - ? -1 - : left.peer > right.peer - ? 1 - : left.counter - right.counter; -} -function compareHistoryRecords(left, right) { - return (left.change.lamport - right.change.lamport || - compareIds(left.change.id, right.change.id)); -} -function compareHistoryOperations(leftChange, leftOperation, rightChange, rightOperation) { - return compareWriter(operationWriter(leftChange, leftOperation), operationWriter(rightChange, rightOperation)); -} -function lowerBoundHistory(records, counter) { - let low = 0; - let high = records.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (records[middle].change.id.counter < counter) - low = middle + 1; - else - high = middle; - } - return low; -} -function lowerBoundOperation(operations, counter) { - let low = 0; - let high = operations.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (operations[middle].counter < counter) - low = middle + 1; - else - high = middle; - } - return low; -} -function lowerBoundIndexedOperation(operations, counter) { - let low = 0; - let high = operations.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (operations[middle].operation.counter < counter) - low = middle + 1; - else - high = middle; - } - return low; -} -function lowerBoundWriter(operations, writer) { - let low = 0; - let high = operations.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (compareWriter(operations[middle].writer, writer) < 0) - low = middle + 1; - else - high = middle; - } - return low; -} -function latestIncludedOperation(history, version) { - if (history === undefined) - return undefined; - let latest; - for (const [peer, operations] of history.byPeer) { - const index = lowerBoundIndexedOperation(operations, version.get(peer) ?? 0) - 1; - const operation = operations[index]; - if (operation !== undefined && - (latest === undefined || compareWriter(latest.writer, operation.writer) < 0)) { - latest = operation; - } - } - return latest; -} -function latestIncludedTreePlacement(history, version, before) { - if (history?.placementsByPeer === undefined) - return undefined; - let latest; - for (const [peer, operations] of history.placementsByPeer) { - const index = Math.min(lowerBoundIndexedOperation(operations, version.get(peer) ?? 0), lowerBoundWriter(operations, before)) - 1; - const operation = operations[index]; - if (operation !== undefined && - (latest === undefined || compareWriter(latest.writer, operation.writer) < 0)) { - latest = operation; - } - } - return latest; -} -function latestIncludedSequenceValue(history, version) { - if (history === undefined) - return undefined; - for (let index = history.length - 1; index >= 0; index -= 1) { - const meta = history[index]; - if (meta.id.counter < (version.get(meta.id.peer) ?? 0)) - return meta; - } - return undefined; -} -function hasSequenceValueMeta(history, writer, peer, counter) { - if (history === undefined) - return false; - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - const meta = history[middle]; - if (compareWriter({ peer: meta.id.peer, lamport: meta.lamport }, writer) < 0) { - low = middle + 1; - } - else { - high = middle; - } - } - const meta = history[low]; - return meta?.id.peer === peer && meta.id.counter === counter; -} -function findSequenceMoveMeta(history, writer) { - const index = findSequenceMoveMetaIndex(history, writer); - return index < 0 ? undefined : history[index]; -} -function findSequenceMoveMetaIndex(history, writer) { - if (history === undefined) - return -1; - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - const meta = history[middle]; - if (compareWriter({ peer: meta.id.peer, lamport: meta.lamport }, writer) < 0) { - low = middle + 1; - } - else { - high = middle; - } - } - return low < history.length ? low : -1; -} -function counterDelta(content) { - if (content.value.type === "double") - return content.value.value; - if (content.value.type === "i64") - return Number(content.value.value); - if (content.value.type === "delta-int") - return content.value.value; - return undefined; -} -function publicChange(change) { - return { - peer: peerIdToString(change.id.peer), - counter: change.id.counter, - lamport: change.lamport, - length: changeLength(change), - timestamp: Number(change.timestamp), - deps: change.dependencies.map(formatOpId), - message: change.message, - }; -} -function historyRecordsToJsonSchema(records, startVersion, withPeerCompression, nullMessage = true) { - const peers = collectJsonPeers(records, startVersion); - const peerMap = withPeerCompression - ? new Map(peers.map((peer, index) => [peer, BigInt(index)])) - : undefined; - return { - schema_version: 1, - start_version: Object.fromEntries(startVersion - .codecEntries() - .map(({ peer, counter }) => [(peerMap?.get(peer) ?? peer).toString(), counter])), - peers: withPeerCompression ? peers.map(peerIdToString) : null, - changes: records.map((record) => historyRecordToJsonChange(record, peerMap, nullMessage)), - }; -} -function historyRecordToJsonChange(record, peerMap, nullMessage) { - const { change } = record; - return { - id: formatJsonOpId(change.id, peerMap), - timestamp: Number(change.timestamp), - deps: change.dependencies.map((id) => formatJsonOpId(id, peerMap)), - lamport: change.lamport, - msg: change.message ?? (nullMessage ? null : undefined), - ops: change.operations.map((operation) => decodedOperationToJson(operation, record.keys, change.id.peer, peerMap)), - }; -} -function collectJsonPeers(records, startVersion) { - const peers = new Set(startVersion._codecEntriesUnsorted().map(({ peer }) => peer)); - const addId = (id) => { - if (id !== undefined) - peers.add(id.peer); - }; - for (const { change } of records) { - addId(change.id); - for (const dependency of change.dependencies) - addId(dependency); - for (const operation of change.operations) { - if (operation.container.kind === "normal") - peers.add(operation.container.peer); - const content = operation.content; - if (content.type === "text-delete" || - content.type === "list-delete" || - content.type === "movable-list-delete") { - addId(content.startId); - } - else if (content.type === "tree-create" || content.type === "tree-move") { - addId(content.subject); - addId(content.parent); - } - else if (content.type === "tree-delete") { - addId(content.subject); - } - else if (content.type === "movable-list-move" || - content.type === "movable-list-set") { - peers.add(content.elementId.peer); - } - } - } - return [...peers].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); -} -function decodedOperationToJson(operation, keys, changePeer, peerMap) { - const operationId = { peer: changePeer, counter: operation.counter }; - const content = operation.content; - let json; - switch (content.type) { - case "map-insert": - json = { - type: "insert", - key: content.key, - value: changeLoroValueToJson(content.value, keys, operationId, peerMap), - }; - break; - case "map-delete": - json = { type: "delete", key: content.key }; - break; - case "text-insert": - json = { type: "insert", pos: content.position, text: content.value }; - break; - case "text-delete": - json = { - type: "delete", - pos: content.position, - len: Number(content.length), - start_id: formatJsonOpId(content.startId, peerMap), - }; - break; - case "text-mark": - json = { - type: "mark", - start: content.start, - end: content.end, - style_key: content.key, - style_value: changeLoroValueToJson(content.value, keys, operationId, peerMap), - info: content.info, - }; - break; - case "text-mark-end": - json = { type: "mark_end" }; - break; - case "list-insert": - case "movable-list-insert": - json = { - type: "insert", - pos: content.position, - value: content.values.map((value, index) => changeLoroValueToJson(value, keys, { peer: changePeer, counter: operation.counter + index }, peerMap)), - }; - break; - case "list-delete": - case "movable-list-delete": - json = { - type: "delete", - pos: content.position, - len: Number(content.length), - start_id: formatJsonOpId(content.startId, peerMap), - }; - break; - case "movable-list-move": - json = { - type: "move", - from: content.from, - to: content.to, - elem_id: formatJsonOpId({ peer: content.elementId.peer, counter: content.elementId.lamport }, peerMap), - }; - break; - case "movable-list-set": - json = { - type: "set", - elem_id: formatJsonOpId({ peer: content.elementId.peer, counter: content.elementId.lamport }, peerMap), - value: changeLoroValueToJson(content.value, keys, operationId, peerMap), - }; - break; - case "tree-create": - case "tree-move": - json = { - type: content.type === "tree-create" ? "create" : "move", - target: formatJsonTreeId(content.subject, peerMap), - parent: content.parent === undefined ? null : formatJsonTreeId(content.parent, peerMap), - fractional_index: bytesToHex(content.position).toUpperCase(), - }; - break; - case "tree-delete": - json = { type: "delete", target: formatJsonTreeId(content.subject, peerMap) }; - break; - case "future": { - const value = content.value; - if (value.type === "double" || value.type === "i64") { - json = { - type: "counter", - value: value.type === "double" ? value.value : Number(value.value), - prop: content.property, - }; - } - else if (value.type === "delta-int") { - json = { type: "counter", value: value.value, prop: content.property }; - } - else { - json = { - type: "unknown", - prop: content.property, - value: changeValueToJsonUnknown(value), - }; - } - break; - } - } - return { - container: formatJsonContainerId(operation.container, peerMap), - content: json, - counter: operation.counter, - }; -} -function changeLoroValueToJson(value, keys, operationId, peerMap) { - switch (value.type) { - case "null": - return null; - case "bool": - case "double": - case "string": - return value.value; - case "i64": - return Number(value.value); - case "binary": - return value.value.slice(); - case "list": - return value.value.map((item, index) => changeLoroValueToJson(item, keys, { peer: operationId.peer, counter: operationId.counter + index }, peerMap)); - case "map": - return Object.fromEntries(value.value.map(([keyIndex, item]) => { - const key = keys[Number(keyIndex)]; - if (key === undefined) - throw new RangeError("JSON value key is out of range"); - return [key, changeLoroValueToJson(item, keys, operationId, peerMap)]; - })); - case "container-type": { - const child = { - kind: "normal", - ...operationId, - containerType: containerTypeFromRawByte(value.value), - }; - return `🦜:${formatJsonContainerId(child, peerMap)}`; - } - } -} -function changeValueToJsonUnknown(value) { - if (typeof value !== "object" || value === null) - return value; - if ("value" in value) { - const inner = value.value; - return typeof inner === "bigint" ? Number(inner) : inner; - } - if ("data" in value) - return value.data.slice(); - return value; -} -function formatJsonOpId(id, peerMap) { - return `${id.counter}@${(peerMap?.get(id.peer) ?? id.peer).toString()}`; -} -function formatJsonTreeId(id, peerMap) { - return formatJsonOpId(id, peerMap); -} -function formatJsonContainerId(id, peerMap) { - return formatContainerId(id.kind === "normal" ? { ...id, peer: peerMap?.get(id.peer) ?? id.peer } : id); -} -function jsonSchemaToHistoryRecords(schema) { - if (!Array.isArray(schema.changes)) - throw new TypeError("JSON changes must be an array"); - const peers = Array.isArray(schema.peers) ? schema.peers.map(parsePeerId) : undefined; - const resolvePeer = (peer) => { - if (peers === undefined) - return peer; - const resolved = peers[Number(peer)]; - if (resolved === undefined) - throw new RangeError(`JSON peer index ${peer} is missing`); - return resolved; - }; - const parseJsonId = (value) => { - const parsed = parseTreeId(value); - return { peer: resolvePeer(parsed.peer), counter: parsed.counter }; - }; - const parseJsonContainer = (value) => { - const parsed = parseContainerId(value); - return parsed.kind === "normal" - ? { ...parsed, peer: resolvePeer(parsed.peer) } - : parsed; - }; - return schema.changes.map((change) => { - const id = parseJsonId(change.id); - const keys = []; - const keyIndices = new Map(); - const registerJsonKey = (key) => { - const existing = keyIndices.get(key); - if (existing !== undefined) - return existing; - const index = keys.length; - keys.push(key); - keyIndices.set(key, index); - return index; - }; - const operations = change.ops.map((operation) => jsonOperationToDecoded(operation, id.peer, resolvePeer, parseJsonContainer, registerJsonKey)); - let expectedCounter = id.counter; - for (const operation of operations) { - if (operation.counter !== expectedCounter) { - throw new RangeError(`JSON operation counter ${operation.counter} does not follow ${expectedCounter}`); - } - expectedCounter += operation.length; - } - return { - change: { - id, - timestamp: BigInt(Math.trunc(change.timestamp)), - dependencies: change.deps.map(parseJsonId), - lamport: change.lamport, - message: change.msg ?? undefined, - operations, - }, - keys, - keyIndices: undefined, - }; - }); -} -function jsonOperationToDecoded(operation, changePeer, resolvePeer, parseJsonContainer, registerKey) { - const container = parseJsonContainer(operation.container); - const containerType = codecTypeToPublic(container.containerType); - const content = operation.content; - const operationId = { peer: changePeer, counter: operation.counter }; - const parseId = (value) => { - if (typeof value !== "string") - throw new TypeError("JSON operation ID must be a string"); - const parsed = parseTreeId(value); - return { peer: resolvePeer(parsed.peer), counter: parsed.counter }; - }; - const parseParent = (value) => value === null || value === undefined ? undefined : parseId(value); - const encodeValue = (value, id = operationId) => jsonValueToChangeLoroValue(value, id, resolvePeer, registerKey); - if (containerType === "Map") { - if (content.type === "delete") { - if (typeof content.key !== "string") - throw new TypeError("map delete key is missing"); - return { - container, - counter: operation.counter, - length: 1, - content: { type: "map-delete", key: content.key }, - }; - } - if (content.type !== "insert" || typeof content.key !== "string") { - throw new TypeError("invalid map JSON operation"); - } - return { - container, - counter: operation.counter, - length: 1, - content: { - type: "map-insert", - key: content.key, - value: encodeValue(content.value), - }, - }; - } - if (containerType === "Text") { - if (content.type === "insert" && typeof content.text === "string") { - return { - container, - counter: operation.counter, - length: Array.from(content.text).length, - content: { - type: "text-insert", - position: requireJsonInteger(content.pos, "text insert position"), - value: content.text, - }, - }; - } - if (content.type === "delete") { - const length = requireJsonInteger(content.len, "text delete length"); - return { - container, - counter: operation.counter, - length: Math.abs(length), - content: { - type: "text-delete", - position: requireJsonInteger(content.pos, "text delete position"), - length: BigInt(length), - startId: parseId(content.start_id), - }, - }; - } - if (content.type === "mark") { - if (typeof content.style_key !== "string") - throw new TypeError("text mark key is missing"); - return { - container, - counter: operation.counter, - length: 1, - content: { - type: "text-mark", - start: requireJsonInteger(content.start, "text mark start"), - end: requireJsonInteger(content.end, "text mark end"), - key: content.style_key, - value: encodeValue(content.style_value), - info: requireJsonInteger(content.info, "text mark info"), - }, - }; - } - if (content.type === "mark_end") { - return { - container, - counter: operation.counter, - length: 1, - content: { type: "text-mark-end" }, - }; - } - throw new TypeError("invalid text JSON operation"); - } - if (containerType === "List" || containerType === "MovableList") { - const movable = containerType === "MovableList"; - if (content.type === "insert") { - if (!Array.isArray(content.value)) - throw new TypeError("list insert value must be an array"); - const values = content.value.map((value, index) => encodeValue(value, { peer: changePeer, counter: operation.counter + index })); - return { - container, - counter: operation.counter, - length: values.length, - content: { - type: movable ? "movable-list-insert" : "list-insert", - position: requireJsonInteger(content.pos, "list insert position"), - values, - }, - }; - } - if (content.type === "delete") { - const length = requireJsonInteger(content.len, "list delete length"); - return { - container, - counter: operation.counter, - length: Math.abs(length), - content: { - type: movable ? "movable-list-delete" : "list-delete", - position: requireJsonInteger(content.pos, "list delete position"), - length: BigInt(length), - startId: parseId(content.start_id), - }, - }; - } - if (movable && content.type === "move") { - const elementId = parseId(content.elem_id); - return { - container, - counter: operation.counter, - length: 1, - content: { - type: "movable-list-move", - from: requireJsonInteger(content.from, "movable-list source"), - to: requireJsonInteger(content.to, "movable-list destination"), - elementId: { peer: elementId.peer, lamport: elementId.counter }, - }, - }; - } - if (movable && content.type === "set") { - const elementId = parseId(content.elem_id); - return { - container, - counter: operation.counter, - length: 1, - content: { - type: "movable-list-set", - elementId: { peer: elementId.peer, lamport: elementId.counter }, - value: encodeValue(content.value), - }, - }; - } - throw new TypeError("invalid list JSON operation"); - } - if (containerType === "Tree") { - if (content.type === "delete") { - return { - container, - counter: operation.counter, - length: 1, - content: { type: "tree-delete", subject: parseId(content.target) }, - }; - } - if (content.type !== "create" && content.type !== "move") { - throw new TypeError("invalid tree JSON operation"); - } - const position = typeof content.fractional_index === "string" - ? hexToBytes(content.fractional_index) - : Uint8Array.of(0x80); - return { - container, - counter: operation.counter, - length: 1, - content: { - type: content.type === "create" ? "tree-create" : "tree-move", - subject: parseId(content.target), - parent: parseParent(content.parent), - position, - }, - }; - } - if (containerType === "Counter") { - const value = typeof content === "number" - ? content - : typeof content.value === "number" - ? content.value - : Number.NaN; - if (!Number.isFinite(value)) - throw new TypeError("counter JSON value must be finite"); - return { - container, - counter: operation.counter, - length: 1, - content: { - type: "future", - property: typeof content === "object" && content !== null && "prop" in content - ? requireJsonInteger(content.prop, "counter property") - : 0, - value: { type: "double", value }, - }, - }; - } - throw new TypeError("JSON updates do not support this container type"); -} -function jsonValueToChangeLoroValue(value, operationId, resolvePeer, registerKey) { - if (value === undefined || value === null) - return { type: "null" }; - if (typeof value === "boolean") - return { type: "bool", value }; - if (typeof value === "number") { - if (!Number.isFinite(value)) - throw new TypeError("JSON update numbers must be finite"); - return Number.isSafeInteger(value) - ? { type: "i64", value: BigInt(value) } - : { type: "double", value }; - } - if (typeof value === "bigint") - return { type: "i64", value }; - if (typeof value === "string") { - if (value.startsWith("🦜:")) { - const rawId = value.slice("🦜:".length); - if (isContainerId(rawId)) { - const child = parseContainerId(rawId); - if (child.kind !== "normal") { - throw new TypeError("JSON child references must use normal container IDs"); - } - const peer = resolvePeer(child.peer); - if (peer !== operationId.peer || child.counter !== operationId.counter) { - throw new RangeError("JSON child container ID does not match its operation ID"); - } - return { - type: "container-type", - value: containerTypeToRawByte(child.containerType), - }; - } - } - return { type: "string", value }; - } - if (value instanceof Uint8Array) - return { type: "binary", value: value.slice() }; - if (Array.isArray(value)) { - return { - type: "list", - value: value.map((item, index) => jsonValueToChangeLoroValue(item, { peer: operationId.peer, counter: operationId.counter + index }, resolvePeer, registerKey)), - }; - } - if (typeof value === "object") { - return { - type: "map", - value: Object.entries(value).map(([key, item]) => [ - BigInt(registerKey(key)), - jsonValueToChangeLoroValue(item, operationId, resolvePeer, registerKey), - ]), - }; - } - throw new TypeError(`unsupported JSON update value type: ${typeof value}`); -} -function requireJsonInteger(value, label) { - if (!Number.isSafeInteger(value)) - throw new TypeError(`${label} must be an integer`); - return value; -} -function unicodeScalarLength(value) { - let length = 0; - for (let index = 0; index < value.length; index += 1) { - const first = value.charCodeAt(index); - if (first >= 0xd800 && - first <= 0xdbff && - index + 1 < value.length && - value.charCodeAt(index + 1) >= 0xdc00 && - value.charCodeAt(index + 1) <= 0xdfff) { - index += 1; - } - length += 1; - } - return length; -} -function compareWriter(left, right) { - return (left.lamport - right.lamport || - (left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0)); -} -function operationWriter(change, operation) { - return { - peer: change.id.peer, - lamport: change.lamport + operation.counter - change.id.counter, - }; -} -function captureBlueprint(container) { - if (container instanceof LoroText) - return { kind: "Text", value: container.toDelta() }; - if (container instanceof LoroCounter) - return { kind: "Counter", value: container.value }; - if (container instanceof LoroList) { - return { - kind: container.kind(), - value: container._visibleElements().map((element) => element.value), - }; - } - return { - kind: container.kind(), - value: container instanceof LoroMap ? container.entries() : container.toJSON(), - }; -} -function restoreBlueprint(container, blueprint) { - if (container instanceof LoroMap) { - for (const [key, value] of blueprint.value) { - if (isContainer(value)) - container.setContainer(key, value); - else - container.set(key, value); - } - } - else if (container instanceof LoroText) { - container.applyDelta(blueprint.value); - } - else if (container instanceof LoroCounter) { - if (blueprint.value !== 0) - container.increment(blueprint.value); - } - else if (container instanceof LoroList) { - for (const value of blueprint.value) { - if (isContainer(value)) - container.pushContainer(value); - else - container.push(value); - } - } -} -function containerDepth(container) { - let depth = 1; - let parent = container.parent(); - while (parent !== undefined) { - depth += 1; - parent = parent.parent(); - } - return depth; -} -function assertTextStyleExpand(value) { - if (value !== "before" && value !== "after" && value !== "none" && value !== "both") { - throw new TypeError(`invalid text style expand mode: ${value}`); - } -} -function textStyleInfoByte(expand, deleting) { - const effective = deleting - ? expand === "none" - ? "both" - : expand === "both" - ? "none" - : expand - : expand; - return (0x80 | - (effective === "before" || effective === "both" ? 0x02 : 0) | - (effective === "after" || effective === "both" ? 0x04 : 0)); -} -function generatePeerId() { - const cryptoObject = globalThis.crypto; - if (cryptoObject !== undefined) { - const bytes = new Uint8Array(8); - cryptoObject.getRandomValues(bytes); - let value = 0n; - for (const byte of bytes) - value = (value << 8n) | BigInt(byte); - if (value !== 0n) - return value; - } - return fallbackPeer++; -} -function importStatus(records, pendingRecords) { - // Group by the bigint peer first so the string form is built once per peer - // rather than once per change. - const spansByPeer = new Map(); - for (const { change } of records) { - const start = change.id.counter; - const end = start + changeLength(change); - const current = spansByPeer.get(change.id.peer); - if (current === undefined) { - spansByPeer.set(change.id.peer, { start, end }); - } - else { - current.start = Math.min(start, current.start); - current.end = Math.max(end, current.end); - } - } - const success = new Map(); - for (const [peer, span] of spansByPeer) { - success.set(peerIdToString(peer), span); - } - const pending = historySpans(pendingRecords); - return { success, pending: pending.size === 0 ? null : pending }; -} -function importStatusBetweenVersions(before, retainedStart, end) { - const success = new Map(); - for (const { peer, counter } of end._codecEntriesUnsorted()) { - const start = Math.max(before.get(peer) ?? 0, retainedStart.get(peer) ?? 0); - if (counter > start) { - success.set(peerIdToString(peer), { start, end: counter }); - } - } - return { success, pending: null }; -} -function historySpans(records) { - const spansByPeer = new Map(); - for (const { change } of records) { - const start = change.id.counter; - const end = start + changeLength(change); - const current = spansByPeer.get(change.id.peer); - if (current === undefined) { - spansByPeer.set(change.id.peer, { start, end }); - } - else { - current.start = Math.min(start, current.start); - current.end = Math.max(end, current.end); - } - } - const spans = new Map(); - for (const [peer, span] of spansByPeer) { - spans.set(peerIdToString(peer), span); - } - return spans; -} -function isEmptyJson(value) { - return (value === "" || - (Array.isArray(value) && value.length === 0) || - (typeof value === "object" && value !== null && Object.keys(value).length === 0) || - value === 0); -} -function containerEventValue(container) { - if (container instanceof LoroMap) - return new Map(container.entries()); - if (container instanceof LoroText) { - return { - text: container.toString(), - delta: container.toDelta(), - }; - } - if (container instanceof LoroTree) { - return container.getNodes().map((node) => ({ - id: node.id, - parent: node.parent()?.id, - index: node.index(), - fractionalIndex: node.fractionalIndex() ?? "", - })); - } - if (container instanceof LoroCounter) - return container.value; - return container.toArray(); -} -function containerDiff(container, beforeValue, mapOperationKeys = { from: undefined, to: undefined }) { - if (container instanceof LoroMap) { - const before = beforeValue instanceof Map ? beforeValue : new Map(); - const after = new Map(container.entries()); - const updated = {}; - const keys = new Set([ - ...before.keys(), - ...after.keys(), - ...(mapOperationKeys.from ?? []), - ...(mapOperationKeys.to ?? []), - ]); - for (const key of keys) { - const previous = before.get(key); - const next = after.get(key); - const tombstonePresenceChanged = !before.has(key) && - !after.has(key) && - mapOperationKeys.from?.has(key) !== mapOperationKeys.to?.has(key); - if (tombstonePresenceChanged || - !eventValuesEqual(previous, next) || - before.has(key) !== after.has(key)) { - updated[key] = next; - } - } - return { type: "map", updated }; - } - if (container instanceof LoroText) { - const before = isTextEventValue(beforeValue) - ? beforeValue - : ({ text: "", delta: [] }); - const after = containerEventValue(container); - if (eventValuesEqual(before.delta, after.delta)) - return { type: "text", diff: [] }; - const styled = [...before.delta, ...after.delta].some((item) => "attributes" in item && item.attributes !== undefined); - if (styled) { - return { - type: "text", - diff: [ - ...(before.text.length === 0 ? [] : [{ delete: before.text.length }]), - ...after.delta, - ], - }; - } - return { type: "text", diff: stringDelta(before.text, after.text) }; - } - if (container instanceof LoroTree) { - return { - type: "tree", - diff: treeDelta(Array.isArray(beforeValue) ? beforeValue : [], containerEventValue(container)), - }; - } - if (container instanceof LoroCounter) { - return { - type: "counter", - increment: container.value - (typeof beforeValue === "number" ? beforeValue : 0), - }; - } - return { - type: "list", - diff: listDelta(Array.isArray(beforeValue) ? beforeValue : [], container.toArray()), - }; -} -function diffForJson(diff) { - if (diff.type === "map") { - return { - type: "map", - updated: Object.fromEntries(Object.entries(diff.updated).map(([key, value]) => [key, jsonDiffValue(value)])), - }; - } - if (diff.type === "list") { - return { - type: "list", - diff: diff.diff.map((operation) => "insert" in operation - ? { ...operation, insert: operation.insert.map(jsonDiffValue) } - : operation), - }; - } - return diff; -} -function jsonDiffValue(value) { - if (isContainer(value)) - return `🦜:${value.id}`; - if (value instanceof Uint8Array || value === null) - return value; - if (Array.isArray(value)) - return value.map(jsonDiffValue); - if (typeof value === "object" && value !== null) { - return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, jsonDiffValue(child)])); - } - return value; -} -// True when a UTF-16 boundary at `at` would split a surrogate pair. -function splitsSurrogatePair(value, at) { - if (at <= 0 || at >= value.length) - return false; - const before = value.charCodeAt(at - 1); - const after = value.charCodeAt(at); - return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff; -} -function stringDelta(before, after) { - // Scan the common prefix/suffix directly in UTF-16 code units and only - // allocate the inserted substring. Boundaries are backed off to code-point - // edges so the delta never splits a surrogate pair. - let prefix = 0; - const prefixLimit = Math.min(before.length, after.length); - while (prefix < prefixLimit && before.charCodeAt(prefix) === after.charCodeAt(prefix)) { - prefix += 1; - } - if (splitsSurrogatePair(before, prefix) || splitsSurrogatePair(after, prefix)) { - prefix -= 1; - } - let suffix = 0; - const suffixLimit = Math.min(before.length - prefix, after.length - prefix); - while (suffix < suffixLimit && - before.charCodeAt(before.length - suffix - 1) === - after.charCodeAt(after.length - suffix - 1)) { - suffix += 1; - } - if (splitsSurrogatePair(before, before.length - suffix) || - splitsSurrogatePair(after, after.length - suffix)) { - suffix -= 1; - } - const diff = []; - if (prefix > 0) - diff.push({ retain: prefix }); - const deleted = before.length - prefix - suffix; - if (deleted > 0) - diff.push({ delete: deleted }); - const inserted = after.slice(prefix, after.length - suffix); - if (inserted.length > 0) - diff.push({ insert: inserted }); - return diff; -} -function listDelta(before, after) { - let prefix = 0; - while (prefix < before.length && - prefix < after.length && - eventValuesEqual(before[prefix], after[prefix])) { - prefix += 1; - } - let suffix = 0; - while (suffix < before.length - prefix && - suffix < after.length - prefix && - eventValuesEqual(before[before.length - suffix - 1], after[after.length - suffix - 1])) { - suffix += 1; - } - const diff = []; - if (prefix > 0) - diff.push({ retain: prefix }); - const deleted = before.length - prefix - suffix; - if (deleted > 0) - diff.push({ delete: deleted }); - const inserted = after.slice(prefix, after.length - suffix); - if (inserted.length > 0) - diff.push({ insert: [...inserted] }); - return diff; -} -function treeDelta(before, after) { - const beforeById = new Map(before.map((node) => [node.id, node])); - const afterById = new Map(after.map((node) => [node.id, node])); - const diff = []; - for (const node of before) { - if (!afterById.has(node.id)) { - diff.push({ - target: node.id, - action: "delete", - oldParent: node.parent, - oldIndex: node.index, - }); - } - } - for (const node of after) { - const previous = beforeById.get(node.id); - if (previous === undefined) { - diff.push({ - target: node.id, - action: "create", - parent: node.parent, - index: node.index, - fractionalIndex: node.fractionalIndex, - }); - } - else if (previous.parent !== node.parent || - previous.index !== node.index || - previous.fractionalIndex !== node.fractionalIndex) { - diff.push({ - target: node.id, - action: "move", - parent: node.parent, - index: node.index, - fractionalIndex: node.fractionalIndex, - oldParent: previous.parent, - oldIndex: previous.index, - }); - } - } - return diff; -} -function eventValuesEqual(left, right) { - if (left === right) - return true; - // undefined compares equal to null, matching how the runtime encodes both - // as the null Loro value. Stored values never contain undefined, so this - // only normalizes user-provided input. - if (left === undefined) - left = null; - if (right === undefined) - right = null; - if (left === right) - return true; - if (left instanceof Uint8Array && right instanceof Uint8Array) { - return bytesEqual(left, right); - } - if (left instanceof Uint8Array || right instanceof Uint8Array) - return false; - if (isContainer(left) && isContainer(right)) - return left.id === right.id; - if (isContainer(left) || isContainer(right)) - return false; - if (Array.isArray(left) && Array.isArray(right)) { - if (left.length !== right.length) - return false; - for (let index = 0; index < left.length; index += 1) { - if (!eventValuesEqual(left[index], right[index])) - return false; - } - return true; - } - if (Array.isArray(left) || Array.isArray(right)) - return false; - if (left instanceof Map && right instanceof Map) { - if (left.size !== right.size) - return false; - for (const [key, value] of left) { - if (!right.has(key) || !eventValuesEqual(value, right.get(key))) - return false; - } - return true; - } - if (left instanceof Map || right instanceof Map) - return false; - if (typeof left === "object" && - left !== null && - typeof right === "object" && - right !== null) { - const leftRecord = left; - const rightRecord = right; - let leftCount = 0; - for (const key in leftRecord) { - leftCount += 1; - if (!(key in rightRecord) || !eventValuesEqual(leftRecord[key], rightRecord[key])) { - return false; - } - } - let rightCount = 0; - for (const _key in rightRecord) - rightCount += 1; - return leftCount === rightCount; - } - return false; -} -function subtractSequenceIdRuns(runs, removedRuns) { - if (removedRuns.length === 0) - return [...runs]; - const removalsByPeer = new Map(); - for (const run of removedRuns) { - let removals = removalsByPeer.get(run.start.peer); - if (removals === undefined) { - removals = []; - removalsByPeer.set(run.start.peer, removals); - } - removals.push({ - start: run.start.counter, - end: run.start.counter + run.length, - }); - } - for (const removals of removalsByPeer.values()) { - removals.sort((left, right) => left.start - right.start); - let write = 0; - for (const removal of removals) { - const previous = write > 0 ? removals[write - 1] : undefined; - if (previous !== undefined && removal.start <= previous.end) { - previous.end = Math.max(previous.end, removal.end); - } - else { - removals[write++] = removal; - } - } - removals.length = write; - } - const retained = []; - for (const run of runs) { - const removals = removalsByPeer.get(run.start.peer); - if (removals === undefined) { - retained.push(run); - continue; - } - const end = run.start.counter + run.length; - let low = 0; - let high = removals.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (removals[middle].end <= run.start.counter) - low = middle + 1; - else - high = middle; - } - let cursor = run.start.counter; - for (let index = low; index < removals.length; index += 1) { - const removal = removals[index]; - if (removal.start >= end) - break; - if (cursor < removal.start) { - retained.push({ - start: { peer: run.start.peer, counter: cursor }, - length: Math.min(removal.start, end) - cursor, - }); - } - cursor = Math.max(cursor, removal.end); - if (cursor >= end) - break; - } - if (cursor < end) { - retained.push({ - start: { peer: run.start.peer, counter: cursor }, - length: end - cursor, - }); - } - } - return retained; -} -function isTextEventValue(value) { - return (typeof value === "object" && - value !== null && - typeof value.text === "string" && - Array.isArray(value.delta)); -} -function isEmptyContainerDiff(diff) { - if (diff.type === "map") - return Object.keys(diff.updated).length === 0; - if (diff.type === "counter") - return diff.increment === 0; - return diff.diff.length === 0; -} -function recoverParentBinding(child, parent) { - if (parent instanceof LoroMap) { - for (const [key, record] of parent._entries) { - if (record.value !== child) - continue; - const binding = { kind: "map", key }; - child._setParentBinding(parent, binding); - return binding; - } - } - else if (parent instanceof LoroList) { - for (const element of parent._sequence.all()) { - if (element.value !== child) - continue; - const binding = { kind: "sequence", element }; - child._setParentBinding(parent, binding); - return binding; - } - } - else if (parent instanceof LoroTree) { - for (const record of parent._nodes.values()) { - if (record.data !== child) - continue; - const binding = { kind: "tree", record }; - child._setParentBinding(parent, binding); - return binding; - } - } - return undefined; -} -function containerPath(container) { - const path = []; - let current = container; - while (current !== undefined) { - const id = current._codecId; - if (id?.kind === "root" && !isMergeableContainerId(id)) { - path.unshift(id.name); - break; - } - const parent = current.parent(); - const binding = current._parentLink?.binding ?? - (parent === undefined ? undefined : recoverParentBinding(current, parent)); - if (parent instanceof LoroMap) { - if (binding?.kind === "map") { - path.unshift(binding.key); - } - } - else if (parent instanceof LoroList) { - const index = binding?.kind === "sequence" - ? parent._sequence.visibleIndexOf(binding.element) - : undefined; - if (index !== undefined && index >= 0) - path.unshift(index); - } - else if (parent instanceof LoroTree) { - if (binding?.kind === "tree") - path.unshift(formatTreeId(binding.record.id)); - else if (id?.kind === "normal") - path.unshift(formatTreeId(id)); - } - current = parent; - } - return path; -} -function parseOptionalPathIndex(value) { - if (!/^(0|[1-9]\d*)$/u.test(value)) - return undefined; - const index = Number(value); - return Number.isSafeInteger(index) ? index : undefined; -} -function parsePathIndex(value) { - return parseOptionalPathIndex(value) ?? -1; -} -function treeNodeAtPath(tree, _parent, part) { - if (part.includes("@")) - return tree.getNodeByID(part); - const index = parseOptionalPathIndex(part); - return index === undefined ? undefined : tree._nodeAt(undefined, index); -} diff --git a/loro-js/src/runtime/ephemeral.js b/loro-js/src/runtime/ephemeral.js deleted file mode 100644 index ff610c09e..000000000 --- a/loro-js/src/runtime/ephemeral.js +++ /dev/null @@ -1,560 +0,0 @@ -import { PostcardReader, PostcardWriter } from "../codec/postcard"; -import { readPostcardValue, writePostcardValue } from "../codec/value"; -import { formatContainerId, isContainerId, parseContainerId, parsePeerId } from "./ids"; -/** The low-level awareness class exported by `loro-crdt`. */ -export class AwarenessWasm { - #peer; - #timeout; - #states = new Map(); - constructor(peer, timeout) { - this.#peer = parsePeerId(peer); - this.#timeout = assertTimeout(timeout); - } - free() { } - encode(peers) { - const now = Date.now(); - const states = []; - for (const peerInput of peers) { - const peer = parsePeerId(peerInput); - const state = this.#states.get(peer); - if (state === undefined || now - state.timestamp > this.#timeout) - continue; - states.push({ peer, counter: state.counter, value: state.value }); - } - return encodeAwarenessStates(states); - } - encodeAll() { - return this.encode(this.peers()); - } - apply(bytes) { - let decoded; - try { - decoded = decodeAwarenessStates(bytes); - } - catch (error) { - throw new Error(`Failed to decode awareness data: ${errorMessage(error)}`); - } - const now = Date.now(); - const updated = []; - const added = []; - for (const state of decoded) { - const current = this.#states.get(state.peer); - if (state.peer === this.#peer || - (current !== undefined && current.counter >= state.counter)) { - continue; - } - this.#states.set(state.peer, { - value: state.value, - counter: state.counter, - timestamp: now, - }); - (current === undefined ? added : updated).push(peerString(state.peer)); - } - return { updated, added }; - } - setLocalState(value) { - const current = this.#states.get(this.#peer); - this.#states.set(this.#peer, { - value: valueToEncoded(value), - counter: (current?.counter ?? 0) + 1, - timestamp: Date.now(), - }); - } - peer() { - return peerString(this.#peer); - } - getAllStates() { - const output = {}; - for (const [peer, state] of this.#states) { - output[peerString(peer)] = encodedToValue(state.value); - } - return output; - } - getState(peer) { - const state = this.#states.get(parsePeerId(peer)); - return state === undefined ? undefined : encodedToValue(state.value); - } - getTimestamp(peer) { - return this.#states.get(parsePeerId(peer))?.timestamp; - } - removeOutdated() { - const now = Date.now(); - const removed = []; - for (const peer of this.orderedPeers()) { - const state = this.#states.get(peer); - if (now - state.timestamp <= this.#timeout) - continue; - this.#states.delete(peer); - removed.push(peerString(peer)); - } - return removed; - } - length() { - return this.#states.size; - } - isEmpty() { - return this.#states.size === 0; - } - peers() { - return this.orderedPeers().map(peerString); - } - orderedPeers() { - const peers = []; - if (this.#states.has(this.#peer)) - peers.push(this.#peer); - for (const peer of this.#states.keys()) { - if (peer !== this.#peer) - peers.push(peer); - } - return peers; - } -} -/** @deprecated Use `EphemeralStore` for new code. */ -export class Awareness { - inner; - #peer; - #timeout; - #listeners = new Set(); - #timer; - constructor(peer, timeout = 30000) { - this.inner = new AwarenessWasm(peer, timeout); - this.#peer = this.inner.peer(); - this.#timeout = timeout; - } - apply(bytes, origin = "remote") { - const { updated, added } = this.inner.apply(bytes); - for (const listener of this.#listeners) { - listener({ updated, added, removed: [] }, origin); - } - this.startTimerIfNotEmpty(); - } - setLocalState(state) { - const wasEmpty = this.inner.getState(this.#peer) === undefined; - this.inner.setLocalState(state); - const peer = this.inner.peer(); - for (const listener of this.#listeners) { - listener(wasEmpty - ? { updated: [], added: [peer], removed: [] } - : { updated: [peer], added: [], removed: [] }, "local"); - } - this.startTimerIfNotEmpty(); - } - getLocalState() { - return this.inner.getState(this.#peer); - } - getAllStates() { - return this.inner.getAllStates(); - } - encode(peers) { - return this.inner.encode(peers); - } - encodeAll() { - return this.inner.encodeAll(); - } - addListener(listener) { - this.#listeners.add(listener); - } - removeListener(listener) { - this.#listeners.delete(listener); - } - peers() { - return this.inner.peers(); - } - destroy() { - if (this.#timer !== undefined) - clearInterval(this.#timer); - this.#timer = undefined; - this.#listeners.clear(); - } - startTimerIfNotEmpty() { - if (this.inner.isEmpty() || this.#timer !== undefined) - return; - this.#timer = setInterval(() => { - const removed = this.inner.removeOutdated(); - if (removed.length > 0) { - for (const listener of this.#listeners) { - listener({ updated: [], added: [], removed }, "timeout"); - } - } - if (!this.inner.isEmpty()) - return; - clearInterval(this.#timer); - this.#timer = undefined; - }, this.#timeout / 2); - } -} -/** The low-level ephemeral key-value store exported by `loro-crdt`. */ -export class EphemeralStoreWasm { - #timeout; - #states = new Map(); - #listeners = new Set(); - #localListeners = new Set(); - #pendingEvents = []; - #pendingLocalUpdates = []; - // Adds made from inside a dispatch are queued and flushed after that - // event's dispatch, preserving the former per-event snapshot semantics - // without copying the listener set per emission. Removals apply to the - // live set immediately; Set iteration skips not-yet-visited removals, - // which matches the old `has` guard. - #queuedListeners = []; - #queuedLocalListeners = []; - #deferringListeners = false; - #deferringLocalListeners = false; - #emittingEvents = false; - #emittingLocalUpdates = false; - constructor(timeout) { - this.#timeout = assertTimeout(timeout); - } - free() { } - set(key, value) { - this.setLocal(key, valueToEncoded(value)); - } - delete(key) { - this.setLocal(key, undefined); - } - get(key) { - const value = this.#states.get(key)?.value; - return value === undefined ? undefined : encodedToValue(value); - } - getAllStates() { - const output = {}; - for (const [key, state] of this.#states) { - if (state.value !== undefined) - output[key] = encodedToValue(state.value); - } - return output; - } - subscribeLocalUpdates(listener) { - if (this.#deferringLocalListeners) - this.#queuedLocalListeners.push(listener); - else - this.#localListeners.add(listener); - return () => { - this.#localListeners.delete(listener); - const queued = this.#queuedLocalListeners.indexOf(listener); - if (queued !== -1) - this.#queuedLocalListeners.splice(queued, 1); - }; - } - subscribe(listener) { - if (this.#deferringListeners) - this.#queuedListeners.push(listener); - else - this.#listeners.add(listener); - return () => { - this.#listeners.delete(listener); - const queued = this.#queuedListeners.indexOf(listener); - if (queued !== -1) - this.#queuedListeners.splice(queued, 1); - }; - } - encode(key) { - const state = this.#states.get(key); - if (state === undefined) - return encodeEphemeralStates([]); - if (Date.now() - state.timestamp > this.#timeout) - return new Uint8Array(); - return encodeEphemeralStates([{ key, ...state }]); - } - encodeAll() { - const now = Date.now(); - const states = []; - for (const [key, state] of this.#states) { - if (now - state.timestamp <= this.#timeout) - states.push({ key, ...state }); - } - return encodeEphemeralStates(states); - } - apply(bytes) { - let decoded; - try { - decoded = decodeEphemeralStates(bytes); - } - catch (error) { - throw new Error(`Failed to decode data: ${errorMessage(error)}`); - } - const added = []; - const updated = []; - const removed = []; - const now = Date.now(); - for (const state of decoded) { - if (now - state.timestamp > this.#timeout) - continue; - const old = this.#states.get(state.key); - if (old !== undefined && old.timestamp >= state.timestamp) - continue; - this.#states.set(state.key, { value: state.value, timestamp: state.timestamp }); - if (old !== undefined && state.value !== undefined) - updated.push(state.key); - else if (old === undefined && state.value !== undefined) - added.push(state.key); - else if (old !== undefined && state.value === undefined) - removed.push(state.key); - } - this.emit({ by: "import", added, updated, removed }); - } - removeOutdated() { - const now = Date.now(); - const removed = []; - for (const [key, state] of this.#states) { - if (now - state.timestamp <= this.#timeout) - continue; - this.#states.delete(key); - if (state.value !== undefined) - removed.push(key); - } - this.emit({ by: "timeout", added: [], updated: [], removed }); - } - isEmpty() { - return this.#states.size === 0; - } - keys() { - const output = []; - for (const [key, state] of this.#states) { - if (state.value !== undefined) - output.push(key); - } - return output; - } - setLocal(key, value) { - const old = this.#states.get(key); - this.#states.set(key, { value, timestamp: Date.now() }); - const bytes = this.encode(key); - this.emitLocalUpdate(bytes); - const isDelete = value === undefined; - this.emit({ - by: "local", - added: old === undefined && !isDelete ? [key] : [], - updated: old !== undefined && !isDelete ? [key] : [], - removed: old !== undefined && isDelete ? [key] : [], - }); - } - emit(event) { - if (this.#emittingEvents) { - this.#pendingEvents.push(event); - return; - } - this.#emittingEvents = true; - const pending = [event]; - try { - while (pending.length > 0) { - const current = pending.pop(); - this.#deferringListeners = true; - try { - for (const listener of this.#listeners) - listener(current); - } - finally { - this.#deferringListeners = false; - for (const listener of this.#queuedListeners) - this.#listeners.add(listener); - this.#queuedListeners.length = 0; - } - pending.push(...this.#pendingEvents.splice(0)); - } - } - finally { - this.#emittingEvents = false; - this.#pendingEvents.length = 0; - } - } - emitLocalUpdate(bytes) { - if (this.#emittingLocalUpdates) { - this.#pendingLocalUpdates.push(bytes); - return; - } - this.#emittingLocalUpdates = true; - const pending = [bytes]; - try { - while (pending.length > 0) { - const current = pending.pop(); - this.#deferringLocalListeners = true; - try { - for (const listener of this.#localListeners) - listener(current); - } - finally { - this.#deferringLocalListeners = false; - for (const listener of this.#queuedLocalListeners) { - this.#localListeners.add(listener); - } - this.#queuedLocalListeners.length = 0; - } - pending.push(...this.#pendingLocalUpdates.splice(0)); - } - } - finally { - this.#emittingLocalUpdates = false; - this.#pendingLocalUpdates.length = 0; - } - } -} -/** A typed, automatically expiring wrapper around `EphemeralStoreWasm`. */ -export class EphemeralStore { - inner; - #timeout; - #timer; - constructor(timeout = 30000) { - this.inner = new EphemeralStoreWasm(timeout); - this.#timeout = timeout; - } - apply(bytes) { - this.inner.apply(bytes); - this.startTimerIfNotEmpty(); - } - set(key, value) { - this.inner.set(key, value); - this.startTimerIfNotEmpty(); - } - delete(key) { - this.inner.delete(key); - } - get(key) { - return this.inner.get(key); - } - getAllStates() { - return this.inner.getAllStates(); - } - encode(key) { - return this.inner.encode(key); - } - encodeAll() { - return this.inner.encodeAll(); - } - keys() { - return this.inner.keys(); - } - destroy() { - if (this.#timer !== undefined) - clearInterval(this.#timer); - this.#timer = undefined; - } - subscribe(listener) { - return this.inner.subscribe(listener); - } - subscribeLocalUpdates(listener) { - return this.inner.subscribeLocalUpdates(listener); - } - startTimerIfNotEmpty() { - if (this.inner.isEmpty() || this.#timer !== undefined) - return; - this.#timer = setInterval(() => { - this.inner.removeOutdated(); - if (!this.inner.isEmpty()) - return; - clearInterval(this.#timer); - this.#timer = undefined; - }, this.#timeout / 2); - } -} -function encodeAwarenessStates(states) { - const writer = new PostcardWriter(); - writer.writeUsize(states.length); - for (const state of states) { - writer.writeU64(state.peer); - writer.writeI32(state.counter); - writePostcardValue(writer, state.value); - } - return writer.toUint8Array(); -} -function decodeAwarenessStates(bytes) { - const reader = new PostcardReader(bytes); - const states = reader.readArray((input) => ({ - peer: input.readU64(), - counter: input.readI32(), - value: readPostcardValue(input), - })); - reader.assertEnd(); - return states; -} -function encodeEphemeralStates(states) { - const writer = new PostcardWriter(); - writer.writeUsize(states.length); - for (const state of states) { - writer.writeString(state.key); - writer.writeOption(state.value, writePostcardValue); - writer.writeI64(BigInt(state.timestamp)); - } - return writer.toUint8Array(); -} -function decodeEphemeralStates(bytes) { - const reader = new PostcardReader(bytes); - const states = reader.readArray((input) => ({ - key: input.readString(), - value: input.readOption(readPostcardValue), - timestamp: Number(input.readI64()), - })); - reader.assertEnd(); - return states; -} -function valueToEncoded(value, depth = 0) { - if (depth > 512) - throw new RangeError("LoroValue nesting depth exceeded"); - if (value === null || value === undefined) - return { type: "null" }; - if (typeof value === "boolean") - return { type: "bool", value }; - if (typeof value === "number") { - return Number.isInteger(value) && value >= -(2 ** 53) && value <= 2 ** 53 - ? { type: "i64", value: BigInt(value) } - : { type: "double", value }; - } - if (typeof value === "string") { - return isContainerId(value) - ? { type: "container", value: parseContainerId(value) } - : { type: "string", value }; - } - if (value instanceof Uint8Array) - return { type: "binary", value: value.slice() }; - if (Array.isArray(value)) { - return { type: "list", value: value.map((item) => valueToEncoded(item, depth + 1)) }; - } - if (value instanceof Map) { - return { - type: "map", - value: [...value].map(([key, item]) => { - if (typeof key !== "string") - throw new TypeError("Map keys must be strings"); - return [key, valueToEncoded(item, depth + 1)]; - }), - }; - } - if (typeof value === "object") { - return { - type: "map", - value: Object.entries(value).map(([key, item]) => [key, valueToEncoded(item, depth + 1)]), - }; - } - throw new TypeError(`unsupported LoroValue type: ${typeof value}`); -} -function encodedToValue(value) { - switch (value.type) { - case "null": - return null; - case "bool": - case "double": - case "string": - return value.value; - case "i64": - return Number(value.value); - case "binary": - return value.value.slice(); - case "list": - return value.value.map(encodedToValue); - case "map": - return Object.fromEntries(value.value.map(([key, item]) => [key, encodedToValue(item)])); - case "container": - return formatContainerId(value.value); - } -} -function assertTimeout(timeout) { - if (!Number.isFinite(timeout)) - throw new TypeError("timeout must be finite"); - return timeout; -} -function peerString(peer) { - return peer.toString(); -} -function errorMessage(error) { - return error instanceof Error ? error.message : String(error); -} diff --git a/loro-js/src/runtime/event-diff.js b/loro-js/src/runtime/event-diff.js deleted file mode 100644 index e96ae5fc1..000000000 --- a/loro-js/src/runtime/event-diff.js +++ /dev/null @@ -1,371 +0,0 @@ -export class SequenceEventDiff { - #kind; - #originalLength; - #root; - #randomState = 2135587861; - constructor(kind, originalLength) { - this.#kind = kind; - this.#originalLength = originalLength; - this.#root = - originalLength === 0 - ? undefined - : this.#newNode({ - kind: "original", - start: 0, - length: originalLength, - attributes: undefined, - }); - } - get length() { - return nodeLength(this.#root); - } - insertText(position, value, attributes) { - if (this.#kind !== "text") - throw new Error("cannot insert text into a list diff"); - if (value.length === 0) - return; - const [left, right] = this.#splitAt(position); - const inserted = this.#newNode({ - kind: "text", - value, - attributes: attributes !== undefined && hasAttributes(attributes) ? attributes : undefined, - }); - this.#root = mergeNodes(mergeNodes(left, inserted), right); - } - insertList(position, value) { - if (this.#kind !== "list") - throw new Error("cannot insert a list into a text diff"); - if (value.length === 0) - return; - const [left, right] = this.#splitAt(position); - const inserted = this.#newNode({ kind: "list", value: [...value] }); - this.#root = mergeNodes(mergeNodes(left, inserted), right); - } - delete(position, length) { - if (length === 0) - return; - const [left, rest] = this.#splitAt(position); - this.#root = rest; - const [, right] = this.#splitAt(length); - this.#root = mergeNodes(left, right); - } - formatText(position, length, key, value) { - if (this.#kind !== "text") - throw new Error("cannot format a list diff"); - if (length === 0) - return; - const [left, rest] = this.#splitAt(position); - this.#root = rest; - const [selected, right] = this.#splitAt(length); - visitNodes(selected, (node) => { - const piece = node.piece; - if (piece.kind === "list") - throw new Error("text diff contains a list piece"); - const attributes = { ...piece.attributes, [key]: value }; - node.piece = - piece.kind === "original" - ? { - kind: "original", - start: piece.start, - length: piece.length, - attributes, - } - : { kind: "text", value: piece.value, attributes }; - }); - this.#root = mergeNodes(mergeNodes(left, selected), right); - } - toDiff() { - return this.#kind === "text" - ? { type: "text", diff: this.#textDelta() } - : { type: "list", diff: this.#listDelta() }; - } - #textDelta() { - const output = []; - let originalCursor = 0; - let inserted = []; - const flushInserted = () => { - let values = []; - let attributes; - const flushGroup = () => { - if (values.length === 0) - return; - appendTextDelta(output, { - insert: values.join(""), - ...(attributes === undefined ? {} : { attributes }), - }); - values = []; - }; - for (const piece of inserted) { - if (values.length > 0 && !attributesEqual(attributes, piece.attributes)) { - flushGroup(); - } - attributes = piece.attributes; - values.push(piece.value); - } - flushGroup(); - inserted = []; - }; - for (const piece of pieces(this.#root)) { - if (piece.kind === "text") { - inserted.push(piece); - continue; - } - if (piece.kind === "list") - throw new Error("text diff contains a list piece"); - if (piece.start < originalCursor) { - throw new Error("text diff original pieces are out of order"); - } - if (piece.start > originalCursor) { - appendTextDelta(output, { delete: piece.start - originalCursor }); - } - flushInserted(); - appendTextDelta(output, { - retain: piece.length, - ...(piece.attributes === undefined ? {} : { attributes: piece.attributes }), - }); - originalCursor = piece.start + piece.length; - } - if (originalCursor < this.#originalLength) { - appendTextDelta(output, { delete: this.#originalLength - originalCursor }); - } - flushInserted(); - trimPlainTrailingRetains(output); - return output; - } - #listDelta() { - const output = []; - let originalCursor = 0; - let inserted = []; - const flushInserted = () => { - if (inserted.length > 0) - appendListDelta(output, { insert: inserted }); - inserted = []; - }; - for (const piece of pieces(this.#root)) { - if (piece.kind === "list") { - inserted.push(...piece.value); - continue; - } - if (piece.kind === "text") - throw new Error("list diff contains a text piece"); - if (piece.start < originalCursor) { - throw new Error("list diff original pieces are out of order"); - } - if (piece.start > originalCursor) { - appendListDelta(output, { delete: piece.start - originalCursor }); - } - flushInserted(); - appendListDelta(output, { retain: piece.length }); - originalCursor = piece.start + piece.length; - } - if (originalCursor < this.#originalLength) { - appendListDelta(output, { delete: this.#originalLength - originalCursor }); - } - flushInserted(); - trimPlainTrailingRetains(output); - return output; - } - #splitAt(position) { - if (!Number.isSafeInteger(position) || position < 0 || position > this.length) { - throw new RangeError(`event diff position ${position} is out of range`); - } - const result = splitNode(this.#root, position, (piece) => this.#newNode(piece)); - this.#root = undefined; - return result; - } - #newNode(piece) { - this.#randomState ^= this.#randomState << 13; - this.#randomState ^= this.#randomState >>> 17; - this.#randomState ^= this.#randomState << 5; - return { - piece, - priority: this.#randomState >>> 0, - left: undefined, - right: undefined, - length: pieceLength(piece), - }; - } -} -function pieceLength(piece) { - if (piece.kind === "original") - return piece.length; - return piece.value.length; -} -function splitPiece(piece, position) { - if (piece.kind === "original") { - return [ - { - kind: "original", - start: piece.start, - length: position, - attributes: piece.attributes, - }, - { - kind: "original", - start: piece.start + position, - length: piece.length - position, - attributes: piece.attributes, - }, - ]; - } - if (piece.kind === "text") { - return [ - { kind: "text", value: piece.value.slice(0, position), attributes: piece.attributes }, - { kind: "text", value: piece.value.slice(position), attributes: piece.attributes }, - ]; - } - return [ - { kind: "list", value: piece.value.slice(0, position) }, - { kind: "list", value: piece.value.slice(position) }, - ]; -} -function nodeLength(node) { - return node?.length ?? 0; -} -function recomputeNode(node) { - node.length = nodeLength(node.left) + pieceLength(node.piece) + nodeLength(node.right); -} -function mergeNodes(left, right) { - if (left === undefined) - return right; - if (right === undefined) - return left; - if (left.priority <= right.priority) { - left.right = mergeNodes(left.right, right); - recomputeNode(left); - return left; - } - right.left = mergeNodes(left, right.left); - recomputeNode(right); - return right; -} -function splitNode(root, position, newNode) { - if (root === undefined) - return [undefined, undefined]; - const leftLength = nodeLength(root.left); - const ownLength = pieceLength(root.piece); - if (position < leftLength) { - const [left, right] = splitNode(root.left, position, newNode); - root.left = right; - recomputeNode(root); - return [left, root]; - } - if (position > leftLength + ownLength) { - const [left, right] = splitNode(root.right, position - leftLength - ownLength, newNode); - root.right = left; - recomputeNode(root); - return [root, right]; - } - if (position === leftLength) { - const left = root.left; - root.left = undefined; - recomputeNode(root); - return [left, root]; - } - if (position === leftLength + ownLength) { - const right = root.right; - root.right = undefined; - recomputeNode(root); - return [root, right]; - } - const [leftPiece, rightPiece] = splitPiece(root.piece, position - leftLength); - return [ - mergeNodes(root.left, newNode(leftPiece)), - mergeNodes(newNode(rightPiece), root.right), - ]; -} -function visitNodes(root, visit) { - const stack = []; - let node = root; - while (node !== undefined || stack.length > 0) { - while (node !== undefined) { - stack.push(node); - node = node.left; - } - node = stack.pop(); - visit(node); - node = node.right; - } -} -function pieces(root) { - const output = []; - visitNodes(root, (node) => output.push(node.piece)); - return output; -} -function appendTextDelta(output, operation) { - const previous = output.at(-1); - if (previous !== undefined && sameDeltaKind(previous, operation)) { - if ("delete" in previous && "delete" in operation) { - output[output.length - 1] = { delete: previous.delete + operation.delete }; - return; - } - if ("retain" in previous && - "retain" in operation && - attributesEqual(previous.attributes, operation.attributes)) { - output[output.length - 1] = { - retain: previous.retain + operation.retain, - ...(previous.attributes === undefined ? {} : { attributes: previous.attributes }), - }; - return; - } - if ("insert" in previous && - "insert" in operation && - attributesEqual(previous.attributes, operation.attributes)) { - output[output.length - 1] = { - insert: previous.insert + operation.insert, - ...(previous.attributes === undefined ? {} : { attributes: previous.attributes }), - }; - return; - } - } - output.push(operation); -} -function appendListDelta(output, operation) { - const previous = output.at(-1); - if (previous !== undefined && "delete" in previous && "delete" in operation) { - output[output.length - 1] = { delete: previous.delete + operation.delete }; - } - else if (previous !== undefined && "retain" in previous && "retain" in operation) { - output[output.length - 1] = { retain: previous.retain + operation.retain }; - } - else if (previous !== undefined && "insert" in previous && "insert" in operation) { - for (const item of operation.insert) - previous.insert.push(item); - } - else { - output.push(operation); - } -} -function sameDeltaKind(left, right) { - return (("insert" in left && "insert" in right) || - ("delete" in left && "delete" in right) || - ("retain" in left && "retain" in right)); -} -function attributesEqual(left, right) { - if (left === right) - return true; - if (left === undefined || right === undefined) - return false; - let count = 0; - for (const key in left) { - count += 1; - if (!Object.is(left[key], right[key])) - return false; - } - for (const _key in right) - count -= 1; - return count === 0; -} -function hasAttributes(attributes) { - for (const _key in attributes) - return true; - return false; -} -function trimPlainTrailingRetains(output) { - while (true) { - const last = output.at(-1); - if (last === undefined || !("retain" in last) || last.attributes !== undefined) - return; - output.pop(); - } -} diff --git a/loro-js/src/runtime/fractional-index.js b/loro-js/src/runtime/fractional-index.js deleted file mode 100644 index facf17a84..000000000 --- a/loro-js/src/runtime/fractional-index.js +++ /dev/null @@ -1,92 +0,0 @@ -import { compareBytes } from "../codec/bytes"; -const TERMINATOR = 128; -export function fractionalIndexBetween(lower, upper) { - if (lower === undefined && upper === undefined) { - return Uint8Array.of(TERMINATOR); - } - if (lower === undefined) { - return terminated(newBefore(upper)); - } - if (upper === undefined) { - return terminated(newAfter(lower)); - } - if (compareBytes(lower, upper) >= 0) { - throw new RangeError("fractional index bounds are not ordered"); - } - const between = newBetween(lower, upper); - if (between === undefined) { - throw new RangeError("cannot create a fractional index between the bounds"); - } - return terminated(between); -} -function terminated(bytes) { - const output = new Uint8Array(bytes.length + 1); - output.set(bytes); - output[bytes.length] = TERMINATOR; - return output; -} -function newBefore(bytes) { - for (let index = 0; index < bytes.length; index += 1) { - if (bytes[index] > TERMINATOR) - return bytes.slice(0, index); - if (bytes[index] > 0) { - const output = bytes.slice(0, index + 1); - output[index] = output[index] - 1; - return output; - } - } - throw new RangeError("invalid fractional index"); -} -function newAfter(bytes) { - for (let index = 0; index < bytes.length; index += 1) { - if (bytes[index] < TERMINATOR) - return bytes.slice(0, index); - if (bytes[index] < 255) { - const output = bytes.slice(0, index + 1); - output[index] = output[index] + 1; - return output; - } - } - throw new RangeError("invalid fractional index"); -} -function newBetween(left, right) { - const shorterLength = Math.min(left.length, right.length) - 1; - for (let index = 0; index < shorterLength; index += 1) { - const difference = right[index] - left[index]; - if (difference > 1) { - const output = left.slice(0, index + 1); - output[index] = output[index] + Math.floor(difference / 2); - return output; - } - if (difference === 1) { - const tail = newAfter(left.subarray(index + 1)); - const output = new Uint8Array(index + 1 + tail.length); - output.set(left.subarray(0, index + 1)); - output.set(tail, index + 1); - return output; - } - if (difference < 0) - return undefined; - } - if (left.length < right.length) { - const split = shorterLength + 1; - if (right[split - 1] < TERMINATOR) - return undefined; - const tail = newBefore(right.subarray(split)); - const output = new Uint8Array(split + tail.length); - output.set(right.subarray(0, split)); - output.set(tail, split); - return output; - } - if (left.length > right.length) { - const split = shorterLength + 1; - if (left[split - 1] >= TERMINATOR) - return undefined; - const tail = newAfter(left.subarray(split)); - const output = new Uint8Array(split + tail.length); - output.set(left.subarray(0, split)); - output.set(tail, split); - return output; - } - return undefined; -} diff --git a/loro-js/src/runtime/ids.js b/loro-js/src/runtime/ids.js deleted file mode 100644 index a82419d1e..000000000 --- a/loro-js/src/runtime/ids.js +++ /dev/null @@ -1,151 +0,0 @@ -import { ContainerType as CodecContainerType, } from "../codec/types"; -const U64_MAX = 0xffffffffffffffffn; -export function parsePeerId(peer) { - let value; - try { - value = typeof peer === "bigint" ? peer : BigInt(peer); - } - catch { - throw new TypeError("peer id must be an unsigned 64-bit integer"); - } - if (value < 0n || value > U64_MAX) { - throw new RangeError(`peer id is out of range: ${String(peer)}`); - } - return value; -} -export function peerIdToString(peer) { - return peer.toString(); -} -export function parseOpId(id) { - assertCounter(id.counter); - return { peer: parsePeerId(id.peer), counter: id.counter }; -} -export function formatOpId(id) { - return { peer: peerIdToString(id.peer), counter: id.counter }; -} -export function formatTreeId(id) { - return `${id.counter}@${id.peer}`; -} -export function idStrToId(id) { - return formatOpId(parseTreeId(id)); -} -export function parseTreeId(id) { - const separator = id.indexOf("@"); - if (separator <= 0 || id.indexOf("@", separator + 1) !== -1) { - throw new TypeError(`invalid tree id: ${id}`); - } - const counter = Number(id.slice(0, separator)); - assertCounter(counter); - return { peer: parsePeerId(id.slice(separator + 1)), counter }; -} -export function codecTypeToPublic(type) { - switch (type) { - case CodecContainerType.Text: - return "Text"; - case CodecContainerType.Map: - return "Map"; - case CodecContainerType.List: - return "List"; - case CodecContainerType.Tree: - return "Tree"; - case CodecContainerType.MovableList: - return "MovableList"; - case CodecContainerType.Counter: - return "Counter"; - default: - throw new TypeError(`unsupported container type ${type.value}`); - } -} -export function publicTypeToCodec(type) { - switch (type) { - case "Text": - return CodecContainerType.Text; - case "Map": - return CodecContainerType.Map; - case "List": - return CodecContainerType.List; - case "Tree": - return CodecContainerType.Tree; - case "MovableList": - return CodecContainerType.MovableList; - case "Counter": - return CodecContainerType.Counter; - } -} -export function formatContainerId(id) { - const type = codecTypeToPublic(id.containerType); - return (id.kind === "root" - ? `cid:root-${id.name}:${type}` - : `cid:${id.counter}@${id.peer}:${type}`); -} -export function parseContainerId(id) { - if (!id.startsWith("cid:")) { - throw new TypeError(`invalid container id: ${id}`); - } - const typeSeparator = id.lastIndexOf(":"); - if (typeSeparator <= 4) { - throw new TypeError(`invalid container id: ${id}`); - } - const type = id.slice(typeSeparator + 1); - const containerType = publicTypeToCodec(type); - const body = id.slice(4, typeSeparator); - if (body.startsWith("root-")) { - return { kind: "root", name: body.slice(5), containerType }; - } - const operationId = parseTreeId(body); - return { kind: "normal", ...operationId, containerType }; -} -export function newContainerID(id, type) { - return formatContainerId({ - kind: "normal", - ...parseOpId(id), - containerType: publicTypeToCodec(type), - }); -} -export function newRootContainerID(name, type) { - return formatContainerId({ - kind: "root", - name, - containerType: publicTypeToCodec(type), - }); -} -export function isContainerId(value) { - // Cheap rejection first: allocating the TypeError (and its stack frames) is - // the dominant cost when this guards arbitrary strings. - if (!value.startsWith("cid:")) - return false; - try { - parseContainerId(value); - return true; - } - catch { - return false; - } -} -export function idsEqual(left, right) { - return left.peer === right.peer && left.counter === right.counter; -} -export function containerIdsEqual(left, right) { - // Structural comparison: stringifying both ids would allocate two throwaway - // strings per call on the B4 edit hot path. - if (left.kind !== right.kind || - !codecContainerTypesEqual(left.containerType, right.containerType)) { - return false; - } - return left.kind === "root" && right.kind === "root" - ? left.name === right.name - : left.kind === "normal" && - right.kind === "normal" && - left.peer === right.peer && - left.counter === right.counter; -} -function codecContainerTypesEqual(left, right) { - return typeof left === "string" || typeof right === "string" - ? left === right - : left.value === right.value; -} -function assertCounter(counter) { - if (!Number.isSafeInteger(counter) || counter < 0 || counter > 2147483647) { - throw new RangeError(`counter is out of range: ${counter}`); - } -} diff --git a/loro-js/src/runtime/index.js b/loro-js/src/runtime/index.js deleted file mode 100644 index a3f1908d2..000000000 --- a/loro-js/src/runtime/index.js +++ /dev/null @@ -1,8 +0,0 @@ -export * from "./containers"; -export * from "./document"; -export * from "./ephemeral"; -export { idStrToId, isContainerId, newContainerID, newRootContainerID } from "./ids"; -export * from "./json-updates"; -export * from "./types"; -export * from "./undo"; -export * from "./version-vector"; diff --git a/loro-js/src/runtime/json-updates.js b/loro-js/src/runtime/json-updates.js deleted file mode 100644 index 8812984e1..000000000 --- a/loro-js/src/runtime/json-updates.js +++ /dev/null @@ -1,60 +0,0 @@ -export function redactJsonUpdates(input, versionRange) { - const parsed = typeof input === "string" ? JSON.parse(input) : input; - const schema = cloneJsonUpdateValue(parsed); - for (const change of schema.changes) { - const peerToken = change.id.slice(change.id.indexOf("@") + 1); - const peer = schema.peers === null ? peerToken : (schema.peers[Number(peerToken)] ?? peerToken); - const range = versionRange[peer]; - if (range === undefined) - continue; - for (const operation of change.ops) - redactOperation(operation, range); - } - return schema; -} -function redactOperation(operation, range) { - const content = operation.content; - if (content.type === "insert" && typeof content.text === "string") { - const characters = Array.from(content.text); - content.text = characters - .map((character, index) => inRange(operation.counter + index, range) ? "�" : character) - .join(""); - return; - } - if (content.type === "insert" && Array.isArray(content.value)) { - content.value = content.value.map((value, index) => inRange(operation.counter + index, range) ? redactValue(value) : value); - return; - } - if (!inRange(operation.counter, range)) - return; - if (content.type === "insert" && typeof content.key === "string") { - content.value = redactValue(content.value); - } - else if (content.type === "set") { - content.value = redactValue(content.value); - } - else if (content.type === "mark") { - content.style_value = null; - } -} -function redactValue(value) { - return typeof value === "string" && value.startsWith("🦜:") - ? value - : null; -} -function inRange(counter, range) { - return counter >= range[0] && counter < range[1]; -} -function cloneJsonUpdateValue(value) { - if (value instanceof Uint8Array) - return value.slice(); - if (Array.isArray(value)) - return value.map(cloneJsonUpdateValue); - if (typeof value === "object" && value !== null) { - return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, cloneJsonUpdateValue(item)])); - } - return value; -} -export function jsonUpdatePeer(schema, compressedPeer) { - return (schema.peers?.[Number(compressedPeer)] ?? compressedPeer); -} diff --git a/loro-js/src/runtime/jsonpath.js b/loro-js/src/runtime/jsonpath.js deleted file mode 100644 index 5e5c49504..000000000 --- a/loro-js/src/runtime/jsonpath.js +++ /dev/null @@ -1,833 +0,0 @@ -import { isContainer, LoroCounter, LoroList, LoroMap, LoroText, LoroTree, } from "./containers"; -export function evaluateJsonPath(root, query) { - const selectors = parseJsonPath(query); - let current = [root]; - for (const selector of selectors) { - current = selector.recursive - ? applyRecursiveSelector(current, selector, root) - : applySelector(current, selector, root); - } - return current; -} -const UNKNOWN_EVENT_INDEX = Symbol("unknownEventIndex"); -export function compileJsonPathEventMatcher(query) { - const steps = parseJsonPath(query).map((selector) => ({ - recursive: selector.recursive, - selectors: selector.kind === "child" - ? selector.keys.map((value) => ({ kind: "key", value })) - : selector.kind === "indices" - ? selector.indices.map((value) => value < 0 - ? { kind: "wildcard", value: undefined } - : { kind: "index", value }) - : [{ kind: "wildcard", value: undefined }], - })); - // Positions stay within [0, steps.length], so membership checks on the - // ping-pong arrays replace the former per-element Set + spread + sort. - const positionsAfter = (path, extraElement) => { - let positions = [0]; - let next = []; - const elementCount = extraElement === undefined ? path.length : path.length + 1; - for (let index = 0; index < elementCount; index += 1) { - const element = index < path.length ? path[index] : extraElement; - next.length = 0; - for (const position of positions) { - if (position >= steps.length) { - if (!next.includes(position)) - next.push(position); - continue; - } - const step = steps[position]; - if (step.recursive && !next.includes(position)) - next.push(position); - if (step.selectors.some((selector) => eventSelectorMatches(selector, element)) && - !next.includes(position + 1)) { - next.push(position + 1); - } - } - if (next.length === 0) - return next; - const swap = positions; - positions = next; - next = swap; - } - return positions; - }; - const mayMatch = (path) => steps.length === 0 || - positionsAfter(path).some((position) => position >= steps.length); - return (event) => { - const basePath = event.path; - if (mayMatch(basePath)) - return true; - if (event.diff.type === "map") { - const basePositions = positionsAfter(basePath); - if (basePositions.length === 0) - return false; - const passedWildcard = basePositions.some((position) => position > 0 && - steps[position - 1]?.selectors.some((selector) => selector.kind === "wildcard")); - for (const key in event.diff.updated) { - if (passedWildcard || positionsAfter(basePath, key).length > 0) - return true; - } - return false; - } - if (event.diff.type === "list" || - event.diff.type === "tree" || - event.diff.type === "counter") { - return positionsAfter(basePath, UNKNOWN_EVENT_INDEX).length > 0; - } - return false; - }; -} -function eventSelectorMatches(selector, element) { - if (selector.kind === "wildcard") - return true; - if (selector.kind === "key") - return typeof element === "string" && element === selector.value; - return element === UNKNOWN_EVENT_INDEX || element === selector.value; -} -function parseJsonPath(query) { - if (typeof query !== "string" || query.length === 0 || query[0] !== "$") { - throw new TypeError("JSONPath must start with '$'"); - } - const selectors = []; - let index = 1; - while (index < query.length) { - let recursive = false; - if (query.startsWith("..", index)) { - recursive = true; - index += 2; - } - else if (query[index] === ".") { - index += 1; - } - else if (query[index] !== "[") { - throw new SyntaxError(`unexpected JSONPath token at ${index}`); - } - if (query[index] === "[") { - const bracket = readBracket(query, index); - selectors.push(parseBracketSelector(bracket.content, recursive)); - index = bracket.end; - continue; - } - if (query[index] === "*") { - selectors.push({ kind: "wildcard", recursive }); - index += 1; - continue; - } - const start = index; - while (index < query.length && /[A-Za-z0-9_$-]/u.test(query[index])) { - index += 1; - } - if (index === start) - throw new SyntaxError(`missing JSONPath child at ${index}`); - selectors.push({ kind: "child", keys: [query.slice(start, index)], recursive }); - } - return selectors; -} -function readBracket(input, start) { - let depth = 0; - let quote; - let escaped = false; - for (let index = start; index < input.length; index += 1) { - const character = input[index]; - if (quote !== undefined) { - if (escaped) - escaped = false; - else if (character === "\\") - escaped = true; - else if (character === quote) - quote = undefined; - continue; - } - if (character === "'" || character === '"') { - quote = character; - continue; - } - if (character === "[") - depth += 1; - else if (character === "]") { - depth -= 1; - if (depth === 0) { - return { content: input.slice(start + 1, index), end: index + 1 }; - } - } - } - throw new SyntaxError(`unterminated JSONPath bracket at ${start}`); -} -function parseBracketSelector(content, recursive) { - const trimmed = content.trim(); - if (trimmed === "*") - return { kind: "wildcard", recursive }; - if (trimmed.startsWith("?(") && trimmed.endsWith(")")) { - return { - kind: "filter", - expression: trimmed.slice(2, -1), - recursive, - }; - } - const parts = splitTopLevel(trimmed, ","); - if (parts.length > 1) { - if (parts.every((part) => isQuoted(part.trim()))) { - return { - kind: "child", - keys: parts.map((part) => parseQuotedString(part.trim())), - recursive, - }; - } - if (parts.every((part) => /^-?\d+$/u.test(part.trim()))) { - return { - kind: "indices", - indices: parts.map((part) => Number(part.trim())), - recursive, - }; - } - throw new SyntaxError("JSONPath union must contain only keys or only indexes"); - } - if (isQuoted(trimmed)) { - return { kind: "child", keys: [parseQuotedString(trimmed)], recursive }; - } - if (/^-?\d+$/u.test(trimmed)) { - return { kind: "indices", indices: [Number(trimmed)], recursive }; - } - const slices = splitTopLevel(trimmed, ":"); - if (slices.length >= 2 && slices.length <= 3) { - const parsePart = (part) => { - const value = part.trim(); - if (value === "") - return undefined; - if (!/^-?\d+$/u.test(value)) - throw new SyntaxError("invalid JSONPath slice"); - return Number(value); - }; - return { - kind: "slice", - start: parsePart(slices[0]), - end: parsePart(slices[1]), - step: slices[2] === undefined ? undefined : parsePart(slices[2]), - recursive, - }; - } - throw new SyntaxError(`unsupported JSONPath selector [${content}]`); -} -function applyRecursiveSelector(values, selector, root) { - const output = []; - // `applySelector` never reads `selector.recursive`, so the selector passes - // through unchanged instead of being cloned per visited node. The ancestor - // set is reused in place: add on the way down, delete on the way up, which - // keeps the same path-scoped cycle detection. - const ancestors = new Set(); - const holder = [undefined]; - const visit = (value) => { - holder[0] = value; - output.push(...applySelector(holder, selector, root)); - const object = objectIdentity(value); - if (object === undefined) { - for (const child of childValues(value)) - visit(child); - return; - } - if (ancestors.has(object)) - return; - ancestors.add(object); - for (const child of childValues(value)) - visit(child); - ancestors.delete(object); - }; - for (const value of values) - visit(value); - return output; -} -function applySelector(values, selector, root) { - const output = []; - switch (selector.kind) { - case "child": - for (const value of values) { - for (const key of selector.keys) { - const child = lookupChild(value, key); - if (child.found) - output.push(child.value); - } - } - return output; - case "wildcard": - return values.flatMap(childValues); - case "indices": - for (const value of values) { - if (value instanceof LoroList) { - for (const rawIndex of selector.indices) { - const index = rawIndex < 0 ? value.length + rawIndex : rawIndex; - if (index >= 0 && index < value.length) - output.push(value.get(index)); - } - continue; - } - if (value instanceof LoroTree) { - for (const rawIndex of selector.indices) { - const selected = value._rootJsonValueAt(rawIndex); - if (selected !== undefined) - output.push(selected); - } - continue; - } - const children = sequenceValues(value); - if (children === undefined) - continue; - for (const rawIndex of selector.indices) { - const index = rawIndex < 0 ? children.length + rawIndex : rawIndex; - if (index >= 0 && index < children.length) - output.push(children[index]); - } - } - return output; - case "slice": - for (const value of values) { - if (value instanceof LoroList) { - output.push(...sliceIndexedValues(value.length, (index) => value.get(index), (start, end) => value._valuesRange(start, end), selector.start, selector.end, selector.step)); - continue; - } - if (value instanceof LoroTree) { - output.push(...sliceIndexedValues(value._rootCount(), (index) => value._rootJsonValueAt(index), (start, end) => value._rootJsonValuesRange(start, end), selector.start, selector.end, selector.step)); - continue; - } - const children = sequenceValues(value); - if (children === undefined) - continue; - output.push(...sliceValues(children, selector.start, selector.end, selector.step)); - } - return output; - case "filter": { - const expression = new FilterParser(selector.expression, root); - for (const value of values) { - for (const child of childValues(value)) { - if (expression.test(child)) - output.push(child); - } - } - return output; - } - } -} -function lookupChild(value, key) { - if (value instanceof LoroMap) { - const record = value._entries.get(key); - if (record === undefined || record.deleted) { - return { found: false, value: undefined }; - } - return { found: true, value: value.get(key) }; - } - if (value instanceof Map) { - return value.has(key) - ? { found: true, value: value.get(key) } - : { found: false, value: undefined }; - } - if (value instanceof LoroList) { - if (!/^-?\d+$/u.test(key)) - return { found: false, value: undefined }; - const rawIndex = Number(key); - const index = rawIndex < 0 ? value.length + rawIndex : rawIndex; - return index >= 0 && index < value.length - ? { found: true, value: value.get(index) } - : { found: false, value: undefined }; - } - if (value instanceof LoroTree) { - if (!/^-?\d+$/u.test(key)) - return { found: false, value: undefined }; - const selected = value._rootJsonValueAt(Number(key)); - return selected === undefined - ? { found: false, value: undefined } - : { found: true, value: selected }; - } - if (Array.isArray(value)) { - if (!/^-?\d+$/u.test(key)) - return { found: false, value: undefined }; - const rawIndex = Number(key); - const index = rawIndex < 0 ? value.length + rawIndex : rawIndex; - return index >= 0 && index < value.length - ? { found: true, value: value[index] } - : { found: false, value: undefined }; - } - if (typeof value === "object" && value !== null && !isContainer(value)) { - return Object.prototype.hasOwnProperty.call(value, key) - ? { found: true, value: value[key] } - : { found: false, value: undefined }; - } - return { found: false, value: undefined }; -} -function childValues(value) { - if (value instanceof LoroMap) - return value.values(); - if (value instanceof Map) - return [...value.values()]; - if (value instanceof LoroList) - return value.toArray(); - if (value instanceof LoroTree) - return value.toJSON(); - if (Array.isArray(value)) - return [...value]; - if (typeof value === "object" && value !== null && !isContainer(value)) { - return Object.values(value); - } - return []; -} -function sequenceValues(value) { - if (value instanceof LoroList) - return value.toArray(); - if (value instanceof LoroTree) - return value.toJSON(); - return Array.isArray(value) ? [...value] : undefined; -} -function sliceValues(values, rawStart, rawEnd, rawStep) { - const step = rawStep ?? 1; - if (step === 0) - return []; - const length = values.length; - const normalize = (value) => { - const resolved = value < 0 ? length + value : value; - return Math.max(step > 0 ? 0 : -1, Math.min(step > 0 ? length : length - 1, resolved)); - }; - const start = rawStart === undefined ? (step > 0 ? 0 : length - 1) : normalize(rawStart); - const end = rawEnd === undefined ? (step > 0 ? length : -1) : normalize(rawEnd); - const output = []; - if (step > 0) { - for (let index = start; index < end; index += step) - output.push(values[index]); - } - else { - for (let index = start; index > end; index += step) - output.push(values[index]); - } - return output; -} -function sliceIndexedValues(length, at, range, rawStart, rawEnd, rawStep) { - const step = rawStep ?? 1; - if (step === 0) - return []; - const normalize = (value) => { - const resolved = value < 0 ? length + value : value; - return Math.max(step > 0 ? 0 : -1, Math.min(step > 0 ? length : length - 1, resolved)); - }; - const start = rawStart === undefined ? (step > 0 ? 0 : length - 1) : normalize(rawStart); - const end = rawEnd === undefined ? (step > 0 ? length : -1) : normalize(rawEnd); - if (step === 1) - return range(start, end); - const output = []; - if (step > 0) { - for (let index = start; index < end; index += step) - output.push(at(index)); - } - else { - for (let index = start; index > end; index += step) - output.push(at(index)); - } - return output; -} -const COMPARISON_OPERATORS = ["==", "!=", "<", "<=", ">", ">=", "contains", "in"]; -const DOUBLE_CHAR_OPERATORS = ["&&", "||", "==", "!=", "<=", ">="]; -const SINGLE_CHAR_OPERATORS = ["!", "<", ">"]; -const PUNCTUATION_CHARS = ["(", ")", "[", "]", ","]; -const FILTER_PATH_TERMINATORS = [")", ",", "!", "<", ">", "="]; -class FilterParser { - #tokens; - #root; - #index = 0; - #current; - constructor(expression, root) { - this.#tokens = tokenizeFilter(expression); - this.#root = root; - } - test(current) { - this.#index = 0; - this.#current = current; - const value = this.#parseOr(); - this.#expect("eof"); - return filterTruthy(value); - } - #parseOr() { - let left = this.#parseAnd(); - while (this.#match("operator", "||")) { - const right = this.#parseAnd(); - left = filterTruthy(left) || filterTruthy(right); - } - return left; - } - #parseAnd() { - let left = this.#parseComparison(); - while (this.#match("operator", "&&")) { - const right = this.#parseComparison(); - left = filterTruthy(left) && filterTruthy(right); - } - return left; - } - #parseComparison() { - const left = this.#parseUnary(); - const token = this.#peek(); - if (token.kind !== "operator" || - !COMPARISON_OPERATORS.includes(token.value)) { - return left; - } - this.#index += 1; - return compareFilterValues(left, this.#parseUnary(), token.value); - } - #parseUnary() { - if (this.#match("operator", "!")) - return !filterTruthy(this.#parseUnary()); - return this.#parsePrimary(); - } - #parsePrimary() { - const token = this.#peek(); - if (token.kind === "literal") { - this.#index += 1; - return token.value; - } - if (token.kind === "path") { - this.#index += 1; - const root = token.value.startsWith("@") ? this.#current : this.#root; - const query = token.value.startsWith("@") - ? `$${token.value.slice(1)}` - : token.value; - return { - kind: "path-result", - values: evaluateJsonPath(root, query), - }; - } - if (token.kind === "identifier") { - this.#index += 1; - const name = token.value; - this.#expect("punctuation", "("); - const args = []; - if (!this.#match("punctuation", ")")) { - do - args.push(this.#parseOr()); - while (this.#match("punctuation", ",")); - this.#expect("punctuation", ")"); - } - return callFilterFunction(name, args); - } - if (this.#match("punctuation", "(")) { - const value = this.#parseOr(); - this.#expect("punctuation", ")"); - return value; - } - if (this.#match("punctuation", "[")) { - const values = []; - if (!this.#match("punctuation", "]")) { - do - values.push(this.#parseOr()); - while (this.#match("punctuation", ",")); - this.#expect("punctuation", "]"); - } - return values; - } - throw new SyntaxError(`unexpected filter token ${token.kind}:${token.value}`); - } - #peek() { - return this.#tokens[this.#index]; - } - #match(kind, value) { - const token = this.#peek(); - if (token.kind !== kind || (value !== undefined && token.value !== value)) - return false; - this.#index += 1; - return true; - } - #expect(kind, value) { - if (!this.#match(kind, value)) { - const token = this.#peek(); - throw new SyntaxError(`expected ${value ?? kind}, got ${token.kind}:${JSON.stringify(token.value)}`); - } - } -} -function tokenizeFilter(expression) { - const tokens = []; - let index = 0; - while (index < expression.length) { - const character = expression[index]; - if (/\s/u.test(character)) { - index += 1; - continue; - } - const pair = expression.slice(index, index + 2); - if (DOUBLE_CHAR_OPERATORS.includes(pair)) { - tokens.push({ kind: "operator", value: pair }); - index += 2; - continue; - } - if (SINGLE_CHAR_OPERATORS.includes(character)) { - tokens.push({ kind: "operator", value: character }); - index += 1; - continue; - } - if (PUNCTUATION_CHARS.includes(character)) { - tokens.push({ kind: "punctuation", value: character }); - index += 1; - continue; - } - if (character === "'" || character === '"') { - const literal = readQuotedLiteral(expression, index); - tokens.push({ kind: "literal", value: literal.value }); - index = literal.end; - continue; - } - if (character === "@" || character === "$") { - const path = readFilterPath(expression, index); - tokens.push({ kind: "path", value: path.value }); - index = path.end; - continue; - } - const number = expression.slice(index).match(/^-?(?:\d+(?:\.\d*)?|\.\d+)/u)?.[0]; - if (number !== undefined) { - tokens.push({ kind: "literal", value: Number(number) }); - index += number.length; - continue; - } - const identifier = expression.slice(index).match(/^[A-Za-z_][A-Za-z0-9_]*/u)?.[0]; - if (identifier !== undefined) { - if (identifier === "true" || identifier === "false") { - tokens.push({ kind: "literal", value: identifier === "true" }); - } - else if (identifier === "null") { - tokens.push({ kind: "literal", value: null }); - } - else if (identifier === "contains" || identifier === "in") { - tokens.push({ kind: "operator", value: identifier }); - } - else { - tokens.push({ kind: "identifier", value: identifier }); - } - index += identifier.length; - continue; - } - throw new SyntaxError(`invalid filter token at ${index}`); - } - tokens.push({ kind: "eof", value: "" }); - return tokens; -} -function readFilterPath(expression, start) { - let index = start + 1; - let bracketDepth = 0; - let quote; - let escaped = false; - while (index < expression.length) { - const character = expression[index]; - if (quote !== undefined) { - if (escaped) - escaped = false; - else if (character === "\\") - escaped = true; - else if (character === quote) - quote = undefined; - index += 1; - continue; - } - if (character === "'" || character === '"') { - quote = character; - index += 1; - continue; - } - if (character === "[") { - bracketDepth += 1; - index += 1; - continue; - } - if (character === "]" && bracketDepth > 0) { - bracketDepth -= 1; - index += 1; - continue; - } - if (bracketDepth === 0 && - (/\s/u.test(character) || FILTER_PATH_TERMINATORS.includes(character))) { - break; - } - index += 1; - } - return { value: expression.slice(start, index), end: index }; -} -function readQuotedLiteral(input, start) { - const quote = input[start]; - let escaped = false; - for (let index = start + 1; index < input.length; index += 1) { - const character = input[index]; - if (escaped) { - escaped = false; - continue; - } - if (character === "\\") { - escaped = true; - continue; - } - if (character === quote) { - return { - value: parseQuotedString(input.slice(start, index + 1)), - end: index + 1, - }; - } - } - throw new SyntaxError(`unterminated string literal at ${start}`); -} -function compareFilterValues(left, right, operator) { - const leftValues = atomValues(left); - const rightValues = atomValues(right); - if (operator === "in") { - const candidates = rightValues.flatMap((value) => collectionValues(value)); - return leftValues.some((value) => candidates.some((candidate) => primitiveCompare(value, candidate, "=="))); - } - return leftValues.some((leftValue) => rightValues.some((rightValue) => primitiveCompare(leftValue, rightValue, operator))); -} -function primitiveCompare(left, right, operator) { - const a = comparableValue(left); - const b = comparableValue(right); - switch (operator) { - case "==": - return Object.is(a, b); - case "!=": - return !Object.is(a, b); - case "contains": - return typeof a === "string" - ? typeof b === "string" && a.includes(b) - : Array.isArray(a) && a.some((value) => Object.is(value, b)); - case "<": - if (typeof a === "number" && typeof b === "number") - return a < b; - if (typeof a === "string" && typeof b === "string") - return a < b; - return false; - case "<=": - if (typeof a === "number" && typeof b === "number") - return a <= b; - if (typeof a === "string" && typeof b === "string") - return a <= b; - return false; - case ">": - if (typeof a === "number" && typeof b === "number") - return a > b; - if (typeof a === "string" && typeof b === "string") - return a > b; - return false; - case ">=": - if (typeof a === "number" && typeof b === "number") - return a >= b; - if (typeof a === "string" && typeof b === "string") - return a >= b; - return false; - default: - throw new SyntaxError(`unsupported filter operator ${operator}`); - } -} -function callFilterFunction(name, args) { - if (name === "count") { - if (args.length !== 1) - throw new TypeError("count() expects one argument"); - return isPathResult(args[0]) - ? args[0].values.length - : collectionValues(args[0]).length; - } - if (name === "value") { - if (args.length !== 1) - throw new TypeError("value() expects one argument"); - return isPathResult(args[0]) ? args[0].values[0] : args[0]; - } - if (name === "length") { - if (args.length !== 1) - throw new TypeError("length() expects one argument"); - const value = isPathResult(args[0]) ? args[0].values[0] : args[0]; - if (typeof value === "string" || Array.isArray(value)) - return value.length; - if (value instanceof LoroMap) - return value.size; - if (value instanceof Map) - return value.size; - if (value instanceof LoroList) - return value.length; - if (typeof value === "object" && value !== null) - return Object.keys(value).length; - return 0; - } - throw new TypeError(`unsupported JSONPath function ${name}()`); -} -function atomValues(value) { - return isPathResult(value) ? [...value.values] : [value]; -} -function collectionValues(value) { - if (Array.isArray(value)) - return value; - if (value instanceof LoroList) - return value.toArray(); - if (value instanceof LoroMap) - return value.values(); - if (value instanceof Map) - return [...value.values()]; - return [value]; -} -function filterTruthy(value) { - if (isPathResult(value)) { - return (value.values.length > 0 && - value.values.some((item) => Boolean(comparableValue(item)))); - } - return Boolean(comparableValue(value)); -} -function comparableValue(value) { - if (value instanceof LoroText) - return value.toString(); - if (value instanceof LoroCounter) - return value.value; - return value; -} -function isPathResult(value) { - return (typeof value === "object" && - value !== null && - value.kind === "path-result"); -} -function objectIdentity(value) { - return typeof value === "object" && value !== null ? value : undefined; -} -function splitTopLevel(input, separator) { - const parts = []; - let start = 0; - let squareDepth = 0; - let roundDepth = 0; - let quote; - let escaped = false; - for (let index = 0; index < input.length; index += 1) { - const character = input[index]; - if (quote !== undefined) { - if (escaped) - escaped = false; - else if (character === "\\") - escaped = true; - else if (character === quote) - quote = undefined; - continue; - } - if (character === "'" || character === '"') - quote = character; - else if (character === "[") - squareDepth += 1; - else if (character === "]") - squareDepth -= 1; - else if (character === "(") - roundDepth += 1; - else if (character === ")") - roundDepth -= 1; - else if (character === separator && squareDepth === 0 && roundDepth === 0) { - parts.push(input.slice(start, index)); - start = index + 1; - } - } - parts.push(input.slice(start)); - return parts; -} -function isQuoted(value) { - return (value.length >= 2 && - ((value[0] === "'" && value.at(-1) === "'") || - (value[0] === '"' && value.at(-1) === '"'))); -} -function parseQuotedString(value) { - if (!isQuoted(value)) - throw new SyntaxError("invalid quoted JSONPath key"); - const quote = value[0]; - const body = value.slice(1, -1); - if (quote === '"') - return JSON.parse(value); - const doubleQuoted = `"${body.replace(/"/gu, '\\"').replace(/\\'/gu, "'")}"`; - return JSON.parse(doubleQuoted); -} diff --git a/loro-js/src/runtime/mergeable.js b/loro-js/src/runtime/mergeable.js deleted file mode 100644 index c451f2344..000000000 --- a/loro-js/src/runtime/mergeable.js +++ /dev/null @@ -1,124 +0,0 @@ -import { containerTypeFromRawByte, containerTypeToRawByte, encodeContainerId, } from "../codec/container-id"; -import { ContainerType, } from "../codec/types"; -const textEncoder = new TextEncoder(); -const MERGEABLE_PREFIX = "🤝:"; -const MARKER_MAGIC = Uint8Array.of(0, 0x4c, 0x4d, 1); -const MARKER_DOMAIN = textEncoder.encode("loro.mergeable.marker.v1"); -export function newMergeableContainerId(parent, key, containerType) { - if (parent.containerType !== ContainerType.Map) { - throw new TypeError("mergeable child parent must be a map"); - } - const parentPayload = parent.kind === "root" && parent.name.startsWith(MERGEABLE_PREFIX) - ? parent.name.slice(MERGEABLE_PREFIX.length) - : parent.kind === "root" - ? `$${escapeSegment(parent.name)}` - : `@${parent.peer.toString(36)}:${signedBase36(parent.counter)}`; - return { - kind: "root", - name: `${MERGEABLE_PREFIX}${parentPayload}>${escapeSegment(key)}`, - containerType, - }; -} -export function mergeableMarker(parent, key, containerType) { - const rawType = containerTypeToRawByte(containerType); - const parentBytes = encodeContainerId(parent); - const keyBytes = textEncoder.encode(key); - const input = new Uint8Array(MARKER_DOMAIN.length + - varintLength(parentBytes.length) + - parentBytes.length + - varintLength(keyBytes.length) + - keyBytes.length + - 1); - let offset = MARKER_DOMAIN.length; - input.set(MARKER_DOMAIN, 0); - offset = writeVarint(input, offset, parentBytes.length); - input.set(parentBytes, offset); - offset += parentBytes.length; - offset = writeVarint(input, offset, keyBytes.length); - input.set(keyBytes, offset); - offset += keyBytes.length; - input[offset] = rawType; - const digest = crc32(input) & 16777215; - return Uint8Array.of(...MARKER_MAGIC, rawType, (digest >>> 16) & 0xff, (digest >>> 8) & 0xff, digest & 0xff); -} -export function parseMergeableMarker(parent, key, value) { - if (!(value instanceof Uint8Array) || value.length !== 8) - return undefined; - for (let index = 0; index < MARKER_MAGIC.length; index += 1) { - if (value[index] !== MARKER_MAGIC[index]) - return undefined; - } - const type = containerTypeFromRawByte(value[4]); - if (typeof type !== "string") - return undefined; - const expected = mergeableMarker(parent, key, type); - for (let index = 0; index < expected.length; index += 1) { - if (value[index] !== expected[index]) - return undefined; - } - return type; -} -export function isMergeableContainerId(id) { - return id.kind === "root" && id.name.startsWith(MERGEABLE_PREFIX); -} -function escapeSegment(value) { - let output = ""; - for (const character of value) { - if (character === "\\") - output += "\\\\"; - else if (character === ">") - output += "\\>"; - else if (character === "/") - output += "\\s"; - else if (character === "\0") - output += "\\0"; - else - output += character; - } - return output; -} -function signedBase36(value) { - return value < 0 ? `-${(-value).toString(36)}` : value.toString(36); -} -function varintLength(value) { - let length = 1; - while (value >= 0x80) { - value = Math.floor(value / 0x80); - length += 1; - } - return length; -} -function writeVarint(output, offset, value) { - do { - let byte = value & 0x7f; - value = Math.floor(value / 0x80); - if (value > 0) - byte |= 0x80; - output[offset] = byte; - offset += 1; - } while (value > 0); - return offset; -} -let crc32Table; -function getCrc32Table() { - if (crc32Table === undefined) { - const table = new Int32Array(256); - for (let byte = 0; byte < 256; byte += 1) { - let crc = byte; - for (let bit = 0; bit < 8; bit += 1) { - crc = (crc >>> 1) ^ (3988292384 & -(crc & 1)); - } - table[byte] = crc; - } - crc32Table = table; - } - return crc32Table; -} -function crc32(bytes) { - const table = getCrc32Table(); - let crc = 4294967295; - for (let index = 0; index < bytes.length; index += 1) { - crc = (crc >>> 8) ^ table[(crc ^ bytes[index]) & 0xff]; - } - return ~crc >>> 0; -} diff --git a/loro-js/src/runtime/ordered-index.js b/loro-js/src/runtime/ordered-index.js deleted file mode 100644 index ce1a2c758..000000000 --- a/loro-js/src/runtime/ordered-index.js +++ /dev/null @@ -1,303 +0,0 @@ -export class OrderedIndex { - #root; - #last; - #randomState = 2246822507; - #compare; - #nodes = new WeakMap(); - constructor(compare) { - this.#compare = compare; - } - get size() { - return nodeSize(this.#root); - } - add(value) { - if (this.#nodes.has(value)) - throw new Error("ordered index already contains value"); - if (this.#last !== undefined && this.#compare(this.#last.value, value) < 0) { - const node = this.#newNode(value); - this.#nodes.set(value, node); - this.#root = merge(this.#root, node); - if (this.#root !== undefined) - this.#root.parent = undefined; - this.#last = node; - return; - } - const index = this.lowerBound(value); - const previous = this.at(index); - if (previous !== undefined && this.#compare(previous, value) === 0) { - throw new Error("ordered index keys must be unique"); - } - const node = this.#newNode(value); - this.#nodes.set(value, node); - const [left, right] = split(this.#root, index); - this.#root = merge(merge(left, node), right); - if (this.#root !== undefined) - this.#root.parent = undefined; - if (this.#last === undefined) - this.#last = node; - } - delete(value) { - const node = this.#nodes.get(value); - if (node === undefined) - return false; - const index = this.#indexOfNode(node); - if (index === undefined) - return false; - const [left, selectedAndRight] = split(this.#root, index); - const [, right] = split(selectedAndRight, 1); - this.#root = merge(left, right); - if (this.#root !== undefined) - this.#root.parent = undefined; - if (this.#last === node) { - this.#last = this.#root === undefined ? undefined : rightmost(this.#root); - } - this.#nodes.delete(value); - return true; - } - at(index) { - if (!Number.isSafeInteger(index) || index < 0 || index >= this.size) - return undefined; - let node = this.#root; - let remaining = index; - while (node !== undefined) { - const leftSize = nodeSize(node.left); - if (remaining < leftSize) - node = node.left; - else if (remaining === leftSize) - return node.value; - else { - remaining -= leftSize + 1; - node = node.right; - } - } - return undefined; - } - indexOf(value) { - const node = this.#nodes.get(value); - if (node === undefined) - return undefined; - return this.#indexOfNode(node); - } - #indexOfNode(node) { - let index = nodeSize(node.left); - let current = node; - while (current.parent !== undefined) { - if (current === current.parent.right) { - index += nodeSize(current.parent.left) + 1; - } - current = current.parent; - } - return current === this.#root ? index : undefined; - } - lowerBound(value) { - return this._lowerBoundBy((current) => this.#compare(current, value)); - } - _lowerBoundBy(compare) { - let node = this.#root; - let offset = 0; - let answer = this.size; - while (node !== undefined) { - if (compare(node.value) < 0) { - offset += nodeSize(node.left) + 1; - node = node.right; - } - else { - answer = offset + nodeSize(node.left); - node = node.left; - } - } - return answer; - } - values() { - const output = []; - const stack = []; - let node = this.#root; - while (node !== undefined || stack.length > 0) { - while (node !== undefined) { - stack.push(node); - node = node.left; - } - node = stack.pop(); - output.push(node.value); - node = node.right; - } - return output; - } - valuesRange(start, end) { - const boundedStart = Math.max(0, Math.min(start, this.size)); - const boundedEnd = Math.max(boundedStart, Math.min(end, this.size)); - const output = []; - collectRange(this.#root, boundedStart, boundedEnd, output); - return output; - } - forEachFrom(lower, visit) { - const stack = []; - let node = this.#root; - while (node !== undefined) { - if (this.#compare(node.value, lower) < 0) { - node = node.right; - } - else { - stack.push(node); - node = node.left; - } - } - while (stack.length > 0) { - node = stack.pop(); - if (visit(node.value) === false) - return; - node = node.right; - while (node !== undefined) { - stack.push(node); - node = node.left; - } - } - } - /** - * In-order iteration starting at rank `index` in O(log n + visited). Skips - * the per-element `at(index)` treap walks of an index loop. An index at or - * beyond `size` visits nothing. - */ - _forEachFromIndex(index, visit) { - const stack = []; - let node = this.#root; - let remaining = index; - while (node !== undefined) { - const leftSize = nodeSize(node.left); - if (remaining < leftSize) { - stack.push(node); - node = node.left; - } - else if (remaining === leftSize) { - stack.push(node); - break; - } - else { - remaining -= leftSize + 1; - node = node.right; - } - } - while (stack.length > 0) { - node = stack.pop(); - if (visit(node.value) === false) - return; - node = node.right; - while (node !== undefined) { - stack.push(node); - node = node.left; - } - } - } - clear() { - this.#root = undefined; - this.#last = undefined; - this.#nodes = new WeakMap(); - } - #newNode(value) { - this.#randomState ^= this.#randomState << 13; - this.#randomState ^= this.#randomState >>> 17; - this.#randomState ^= this.#randomState << 5; - return { - value, - priority: this.#randomState >>> 0, - left: undefined, - right: undefined, - parent: undefined, - size: 1, - }; - } -} -function nodeSize(node) { - return node?.size ?? 0; -} -function rightmost(node) { - while (node.right !== undefined) - node = node.right; - return node; -} -function collectRange(node, start, end, output) { - if (node === undefined || start >= end) - return; - const leftSize = nodeSize(node.left); - if (start < leftSize) { - collectRange(node.left, start, Math.min(end, leftSize), output); - } - if (start <= leftSize && leftSize < end) - output.push(node.value); - if (end > leftSize + 1) { - collectRange(node.right, Math.max(0, start - leftSize - 1), end - leftSize - 1, output); - } -} -function recompute(node) { - node.size = nodeSize(node.left) + 1 + nodeSize(node.right); - if (node.left !== undefined) - node.left.parent = node; - if (node.right !== undefined) - node.right.parent = node; -} -function merge(left, right) { - if (left === undefined) { - if (right !== undefined) - right.parent = undefined; - return right; - } - if (right === undefined) { - left.parent = undefined; - return left; - } - if (left.priority <= right.priority) { - left.right = merge(left.right, right); - recompute(left); - left.parent = undefined; - return left; - } - right.left = merge(left, right.left); - recompute(right); - right.parent = undefined; - return right; -} -function split(root, count) { - if (root === undefined) - return [undefined, undefined]; - // Descend once, recording the spine. A spine node joins the right result - // when the split continues into its left subtree, the left result - // otherwise; the two carries are then stitched bottom-up. This keeps the - // recursive shape without allocating a [left, right] tuple per level. - const spine = []; - const intoLeftSubtree = []; - let node = root; - let remaining = count; - while (node !== undefined) { - const leftSize = nodeSize(node.left); - const intoLeft = leftSize >= remaining; - spine.push(node); - intoLeftSubtree.push(intoLeft); - if (intoLeft) { - node = node.left; - } - else { - remaining -= leftSize + 1; - node = node.right; - } - } - let left; - let right; - for (let index = spine.length - 1; index >= 0; index -= 1) { - const current = spine[index]; - if (intoLeftSubtree[index]) { - current.left = right; - recompute(current); - right = current; - } - else { - current.right = left; - recompute(current); - left = current; - } - } - if (left !== undefined) - left.parent = undefined; - if (right !== undefined) - right.parent = undefined; - return [left, right]; -} diff --git a/loro-js/src/runtime/sequence-deletion-index.js b/loro-js/src/runtime/sequence-deletion-index.js deleted file mode 100644 index 3c860a7cc..000000000 --- a/loro-js/src/runtime/sequence-deletion-index.js +++ /dev/null @@ -1,227 +0,0 @@ -import { OrderedIndex } from "./ordered-index"; -// Reused deletion id passed to predicates so visited mappings do not each -// allocate a short-lived id object. Predicates must not retain the id -// argument or query deletions reentrantly. -const scratchDeletionId = { peer: 0n, counter: 0 }; -// Reused OrderedIndex.forEachFrom probe in `add`; only `.start` is read. -const targetSegmentProbe = { start: 0, end: 0, mappings: [] }; -export class SequenceDeletionIndex { - #targetsByPeer = new Map(); - #operationsByPeer = new Map(); - #serial = 0; - add(targetStart, length, deleteStart, direction) { - if (length <= 0) - return; - const mapping = { - targetPeer: targetStart.peer, - targetStart: targetStart.counter, - deletePeer: deleteStart.peer, - deleteStart: deleteStart.counter, - length, - direction, - }; - const targets = this.#targets(targetStart.peer); - const targetEnd = targetStart.counter + length; - this.#ensureBoundary(targets, targetStart.counter); - this.#ensureBoundary(targets, targetEnd); - const existing = []; - // OrderedIndex.forEachFrom only reads `.start` through the comparator - // and never retains the probe, so a shared scratch object is safe. - targetSegmentProbe.start = targetStart.counter; - targets.forEachFrom(targetSegmentProbe, (segment) => { - if (segment.start >= targetEnd) - return false; - existing.push(segment); - }); - let cursor = targetStart.counter; - for (const segment of existing) { - if (cursor < segment.start) { - targets.add({ start: cursor, end: segment.start, mappings: [mapping] }); - } - addMapping(segment.mappings, mapping); - cursor = segment.end; - } - if (cursor < targetEnd) { - targets.add({ start: cursor, end: targetEnd, mappings: [mapping] }); - } - this.#operations(deleteStart.peer).add({ - targetPeer: mapping.targetPeer, - targetStart: mapping.targetStart, - deletePeer: mapping.deletePeer, - deleteStart: mapping.deleteStart, - length: mapping.length, - direction: mapping.direction, - serial: this.#serial++, - }); - } - deletionIdsAt(target) { - const segment = this.#targetSegmentAt(target); - if (segment === undefined) - return []; - return segment.mappings.map((mapping) => ({ - peer: mapping.deletePeer, - counter: deleteCounterAt(mapping, target.counter), - })); - } - someDeletion(target, predicate) { - const mappings = this.#targetSegmentAt(target)?.mappings; - if (mappings === undefined) - return false; - for (let index = 0; index < mappings.length; index += 1) { - const mapping = mappings[index]; - scratchDeletionId.peer = mapping.deletePeer; - scratchDeletionId.counter = deleteCounterAt(mapping, target.counter); - if (predicate(scratchDeletionId)) - return true; - } - return false; - } - everyDeletion(target, predicate) { - const mappings = this.#targetSegmentAt(target)?.mappings; - if (mappings === undefined) - return true; - for (let index = 0; index < mappings.length; index += 1) { - const mapping = mappings[index]; - scratchDeletionId.peer = mapping.deletePeer; - scratchDeletionId.counter = deleteCounterAt(mapping, target.counter); - if (!predicate(scratchDeletionId)) - return false; - } - return true; - } - targetRunsDeletedBy(peer, start, end) { - if (start >= end) - return []; - const operations = this.#operationsByPeer.get(peer); - if (operations === undefined) - return []; - const selected = []; - const first = Math.max(0, operations._lowerBoundBy((operation) => operation.deleteStart - start) - 1); - for (let index = first; index < operations.size; index += 1) { - const operation = operations.at(index); - if (operation.deleteStart >= end) - break; - const overlapStart = Math.max(start, operation.deleteStart); - const overlapEnd = Math.min(end, operation.deleteStart + operation.length); - if (overlapStart >= overlapEnd) - continue; - const targetCounter = operation.direction === 1 - ? operation.targetStart + overlapStart - operation.deleteStart - : operation.targetStart + - operation.length - - (overlapEnd - operation.deleteStart); - selected.push({ - start: { peer: operation.targetPeer, counter: targetCounter }, - length: overlapEnd - overlapStart, - }); - } - return normalizeRuns(selected); - } - hasDeletionAt(runs, version) { - for (const run of runs) { - const targets = this.#targetsByPeer.get(run.start.peer); - if (targets === undefined) - continue; - const start = run.start.counter; - const end = start + run.length; - let index = Math.max(0, targets._lowerBoundBy((segment) => segment.start - (start + 1)) - 1); - for (; index < targets.size; index += 1) { - const segment = targets.at(index); - if (segment.start >= end) - break; - const overlapStart = Math.max(start, segment.start); - const overlapEnd = Math.min(end, segment.end); - if (overlapStart >= overlapEnd) - continue; - for (const mapping of segment.mappings) { - const minimumCounter = mapping.direction === 1 - ? deleteCounterAt(mapping, overlapStart) - : deleteCounterAt(mapping, overlapEnd - 1); - if (minimumCounter < (version.get(mapping.deletePeer) ?? 0)) - return true; - } - } - } - return false; - } - reset() { - this.#targetsByPeer.clear(); - this.#operationsByPeer.clear(); - this.#serial = 0; - } - #targets(peer) { - let targets = this.#targetsByPeer.get(peer); - if (targets === undefined) { - targets = new OrderedIndex((left, right) => left.start - right.start); - this.#targetsByPeer.set(peer, targets); - } - return targets; - } - #operations(peer) { - let operations = this.#operationsByPeer.get(peer); - if (operations === undefined) { - operations = new OrderedIndex((left, right) => left.deleteStart - right.deleteStart || left.serial - right.serial); - this.#operationsByPeer.set(peer, operations); - } - return operations; - } - #targetSegmentAt(target) { - const targets = this.#targetsByPeer.get(target.peer); - if (targets === undefined) - return undefined; - const index = targets._lowerBoundBy((segment) => segment.start - (target.counter + 1)) - 1; - const segment = targets.at(index); - return segment !== undefined && target.counter < segment.end ? segment : undefined; - } - #ensureBoundary(targets, counter) { - const index = targets._lowerBoundBy((segment) => segment.start - counter); - if (targets.at(index)?.start === counter) - return; - const previous = targets.at(index - 1); - if (previous === undefined || counter >= previous.end) - return; - targets.delete(previous); - const right = { - start: counter, - end: previous.end, - mappings: [...previous.mappings], - }; - previous.end = counter; - targets.add(previous); - targets.add(right); - } -} -function addMapping(mappings, mapping) { - if (mappings.some((current) => current.deletePeer === mapping.deletePeer && - current.deleteStart === mapping.deleteStart && - current.targetPeer === mapping.targetPeer && - current.targetStart === mapping.targetStart)) { - return; - } - mappings.push(mapping); -} -function deleteCounterAt(mapping, targetCounter) { - const offset = targetCounter - mapping.targetStart; - return (mapping.deleteStart + (mapping.direction === 1 ? offset : mapping.length - 1 - offset)); -} -function normalizeRuns(runs) { - const sorted = [...runs].sort((left, right) => left.start.peer < right.start.peer - ? -1 - : left.start.peer > right.start.peer - ? 1 - : left.start.counter - right.start.counter); - const merged = []; - for (const run of sorted) { - const previous = merged.at(-1); - if (previous !== undefined && - previous.start.peer === run.start.peer && - run.start.counter <= previous.start.counter + previous.length) { - previous.length = - Math.max(previous.start.counter + previous.length, run.start.counter + run.length) - previous.start.counter; - } - else { - merged.push({ start: { ...run.start }, length: run.length }); - } - } - return merged; -} diff --git a/loro-js/src/runtime/sequence-index.js b/loro-js/src/runtime/sequence-index.js deleted file mode 100644 index 37c924a55..000000000 --- a/loro-js/src/runtime/sequence-index.js +++ /dev/null @@ -1,3173 +0,0 @@ -import { OrderedIndex } from "./ordered-index"; -import { SequenceDeletionIndex } from "./sequence-deletion-index"; -/** - * A compact physical sequence span. Implementations can keep scalar fields in - * columns and only create a T view when an API actually asks for one. - */ -export class SequenceSpan { - peerAt(offset) { - return this.idAt(offset).peer; - } - counterAt(offset) { - return this.idAt(offset).counter; - } - idsAreUnique() { - return false; - } - append(_other) { - return false; - } - deletionIdsAt(_offset) { - return EMPTY_SEQUENCE_IDS; - } - retain(_offset, _element) { } - forEachRetained(_visit) { } -} -const SEQUENCE_NODE = Symbol("sequenceNode"); -const SEQUENCE_OFFSET = Symbol("sequenceOffset"); -const MAX_CACHED_CAUSAL_VIEWS = 8; -const MAX_SEQUENCE_SPAN = 32; -const SEQUENCE_LOCATION_STRIDE = 64; -// Shared list metrics: every receiver reads utf16/utf8 and never mutates. -const LIST_METRICS = { utf16: 1, utf8: 1 }; -const listMetrics = () => LIST_METRICS; -// Shared empty list for spans without deletion metadata; callers only read it. -const EMPTY_SEQUENCE_IDS = []; -export class SequenceIndex { - #root; - #tail; - #structureVersion = 0; - #randomState = 2654435769; - #metrics; - #locationsByPeer = new Map(); - #locationsByLamportPeer = new Map(); - #nodesByLocationId = []; - #deletionsByPeer = new Map(); - #scalarDeletionTargetsByPeer = new Map(); - #deletionRuns = new SequenceDeletionIndex(); - #maxCounterByPeer = new Map(); - #cachedCausalViews = []; - #hasLazyDeletions = false; - constructor(metrics = listMetrics) { - this.#metrics = metrics; - } - get allLength() { - return allCount(this.#root); - } - get visibleLength() { - return visibleCount(this.#root); - } - get visibleUtf16Length() { - return visibleMetric(this.#root, "utf16"); - } - get visibleUtf8Length() { - return visibleMetric(this.#root, "utf8"); - } - get spanCount() { - let count = 0; - visitInOrder(this.#root, () => { - count += 1; - }); - return count; - } - /** Changes whenever the physical order or membership changes. */ - get structureVersion() { - return this.#structureVersion; - } - all() { - if (this.#hasLazyDeletions) { - materializeAllDeletions(this.#root, this.#metrics); - this.#hasLazyDeletions = false; - } - const output = []; - visitInOrder(this.#root, (node) => appendNodeElements(node, output)); - return output; - } - visible() { - const output = []; - this.forEachVisible((element) => { - output.push(element); - }); - return output; - } - forEachVisible(visit) { - if (this.visibleLength === 0) - return; - const stack = []; - let node = this.#root; - while (node !== undefined || stack.length > 0) { - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - stack.push(node); - node = visibleCount(node.left) > 0 ? node.left : undefined; - } - node = stack.pop(); - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (nodeDeleted(node, offset)) - continue; - if (visit(nodeElement(node, offset)) === false) - return; - } - node = visibleCount(node.right) > 0 ? node.right : undefined; - } - } - /** - * Visits maximal visible ranges in each physical storage block without - * materializing scalar views from compact spans. - */ - forEachVisibleStorageRange(visit) { - if (this.visibleLength === 0) - return; - const stack = []; - let node = this.#root; - while (node !== undefined || stack.length > 0) { - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - stack.push(node); - node = visibleCount(node.left) > 0 ? node.left : undefined; - } - node = stack.pop(); - let rangeStart = -1; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (!nodeDeleted(node, offset)) { - if (rangeStart < 0) - rangeStart = offset; - continue; - } - if (rangeStart >= 0) { - if (visit(node.element, rangeStart, offset) === false) - return; - rangeStart = -1; - } - } - if (rangeStart >= 0 && - visit(node.element, rangeStart, nodeLength(node)) === false) { - return; - } - node = visibleCount(node.right) > 0 ? node.right : undefined; - } - } - visibleRange(start, end) { - const output = []; - this.forEachVisibleRange(start, end, (element) => { - output.push(element); - }); - return output; - } - forEachVisibleRange(start, end, visit) { - const boundedStart = Math.max(0, Math.min(start, this.visibleLength)); - const boundedEnd = Math.max(boundedStart, Math.min(end, this.visibleLength)); - visitVisibleRange(this.#root, boundedStart, boundedEnd, this.#metrics, visit); - } - someVisibleRange(start, end, predicate) { - let matched = false; - this.forEachVisibleRange(start, end, (element) => { - if (!predicate(element)) - return; - matched = true; - return false; - }); - return matched; - } - visibleIdRuns(start, end) { - const boundedStart = Math.max(0, Math.min(start, this.visibleLength)); - const boundedEnd = Math.max(boundedStart, Math.min(end, this.visibleLength)); - const runs = []; - const append = (id, length = 1) => { - if (length === 0) - return; - const previous = runs[runs.length - 1]; - if (previous !== undefined && - previous.start.peer === id.peer && - previous.start.counter + previous.length === id.counter) { - previous.length += length; - } - else { - runs.push({ start: { ...id }, length }); - } - }; - visitVisibleIdRuns(this.#root, boundedStart, boundedEnd, this.#metrics, append); - return runs; - } - visibleMetricRangesForIdRuns(runs, metric) { - const targets = counterRangesByPeer(runs); - const ranges = []; - const append = (start, end) => { - if (start >= end) - return; - const previous = ranges.at(-1); - if (previous?.end === start) - previous.end = end; - else - ranges.push({ start, end }); - }; - visitVisibleMetricRangesForIds(this.#root, 0, metric, targets, this.#metrics, append); - return ranges; - } - containsIdRuns(runs) { - if (runs.length === 0) - return true; - const targets = counterRangesByPeer(runs); - const expected = [...targets.values()].reduce((total, ranges) => total + ranges.reduce((subtotal, range) => subtotal + range.end - range.start, 0), 0); - return countPhysicalIdsInRanges(this.#root, targets) === expected; - } - atPhysical(index) { - const element = this.atPhysicalRaw(index); - if (element !== undefined && this.#hasLazyDeletions) { - materializeElementDeletion(element, this.#metrics); - } - return element; - } - atPhysicalRaw(index) { - if (!Number.isSafeInteger(index) || index < 0 || index >= this.allLength) { - return undefined; - } - let node = this.#root; - let remaining = index; - while (node !== undefined) { - const leftCount = allCount(node.left); - if (remaining < leftCount) { - node = node.left; - } - else if (remaining < leftCount + nodeLength(node)) { - return nodeElement(node, remaining - leftCount); - } - else { - remaining -= leftCount + nodeLength(node); - node = node.right; - } - } - return undefined; - } - forEachPhysicalRaw(visit) { - let index = 0; - let stopped = false; - visitInOrder(this.#root, (node) => { - if (stopped) - return; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (visit(nodeElement(node, offset), index) === false) { - stopped = true; - return; - } - index += 1; - } - }); - } - findNextIncludedPhysical(start, version) { - if (!Number.isSafeInteger(start) || start < 0 || start >= this.allLength) { - return undefined; - } - return findNextIncludedPhysical(this.#root, 0, start, version); - } - atVisible(index) { - if (!Number.isSafeInteger(index) || index < 0 || index >= this.visibleLength) { - return undefined; - } - let node = this.#root; - let remaining = index; - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - const leftCount = visibleCount(node.left); - if (remaining < leftCount) { - node = node.left; - continue; - } - remaining -= leftCount; - if (remaining < ownVisibleCount(node)) { - return nodeElement(node, physicalOffsetAtVisibleIndex(node, remaining)); - } - remaining -= ownVisibleCount(node); - node = node.right; - } - return undefined; - } - visibleInsertionContext(position) { - if (!Number.isSafeInteger(position) || - position < 0 || - position > this.visibleLength) { - throw new RangeError(`visible sequence position ${position} is out of range`); - } - if (position === 0) { - return { - current: true, - left: undefined, - startIndex: 0, - right: firstPhysicalElement(this.#root), - }; - } - let node = this.#root; - let remaining = position - 1; - let physicalBase = 0; - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - const leftPhysicalCount = allCount(node.left); - const leftVisibleCount = visibleCount(node.left); - if (remaining < leftVisibleCount) { - node = node.left; - continue; - } - remaining -= leftVisibleCount; - physicalBase += leftPhysicalCount; - if (remaining < ownVisibleCount(node)) { - const offset = physicalOffsetAtVisibleIndex(node, remaining); - const left = nodeElement(node, offset); - return { - current: true, - left, - startIndex: physicalBase + offset + 1, - right: nextPhysicalElement(node, offset), - }; - } - remaining -= ownVisibleCount(node); - physicalBase += nodeLength(node); - node = node.right; - } - throw new Error("visible sequence insertion position is missing"); - } - /** Writes insertion neighbor IDs without creating compact-span views. */ - visibleInsertionIdContext(position, output) { - if (!Number.isSafeInteger(position) || - position < 0 || - position > this.visibleLength) { - throw new RangeError(`visible sequence position ${position} is out of range`); - } - if (position === 0) { - output.leftPeer = undefined; - output.leftCounter = 0; - output.startIndex = 0; - writeFirstPhysicalId(this.#root, output); - return output; - } - let node = this.#root; - let remaining = position - 1; - let physicalBase = 0; - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - const leftPhysicalCount = allCount(node.left); - const leftVisibleCount = visibleCount(node.left); - if (remaining < leftVisibleCount) { - node = node.left; - continue; - } - remaining -= leftVisibleCount; - physicalBase += leftPhysicalCount; - if (remaining < ownVisibleCount(node)) { - const offset = physicalOffsetAtVisibleIndex(node, remaining); - output.leftPeer = nodePeer(node, offset); - output.leftCounter = nodeCounter(node, offset); - output.startIndex = physicalBase + offset + 1; - writeNextPhysicalId(node, offset, output); - return output; - } - remaining -= ownVisibleCount(node); - physicalBase += nodeLength(node); - node = node.right; - } - throw new Error("visible sequence insertion position is missing"); - } - findById(id) { - const element = this.findByIdRaw(id); - if (element !== undefined && this.#hasLazyDeletions) { - materializeElementDeletion(element, this.#metrics); - } - return element; - } - findByIdRaw(id) { - const location = this.#locationsByPeer.get(id.peer)?.get(id.counter); - return location === undefined - ? undefined - : typeof location === "number" - ? this.#elementAtLocation(location) - : location; - } - findByLamport(peer, lamport) { - let lamports = this.#locationsByLamportPeer.get(peer); - if (lamports === undefined) { - const locations = this.#locationsByPeer.get(peer); - if (locations === undefined) - return undefined; - const indexedLamports = new Map(); - locations.forEach(0, Number.MAX_SAFE_INTEGER, (location) => { - const lamport = typeof location === "number" - ? nodeLamport(this.#nodeAtLocation(location), location % SEQUENCE_LOCATION_STRIDE) - : location.lamport; - if (lamport !== undefined) { - indexedLamports.set(lamport, location); - } - }); - lamports = indexedLamports; - this.#locationsByLamportPeer.set(peer, lamports); - } - const location = lamports.get(lamport); - const element = location === undefined - ? undefined - : typeof location === "number" - ? this.#elementAtLocation(location) - : location; - if (element !== undefined && this.#hasLazyDeletions) { - materializeElementDeletion(element, this.#metrics); - } - return element; - } - physicalIndexOf(element) { - const node = elementNode(element); - const offset = elementOffset(element); - if (node === undefined || offset === undefined) - return undefined; - let index = allCount(node.left) + offset; - let current = node; - while (current.parent !== undefined) { - if (current === current.parent.right) { - index += allCount(current.parent.left) + nodeLength(current.parent); - } - current = current.parent; - } - return current === this.#root ? index : undefined; - } - visibleIndexOf(element) { - if (this.#hasLazyDeletions) - materializeElementDeletion(element, this.#metrics); - const node = elementNode(element); - const offset = elementOffset(element); - if (node === undefined || offset === undefined) - return undefined; - let index = visibleCount(node.left) + visibleElementsBefore(node, offset); - let current = node; - while (current.parent !== undefined) { - if (current === current.parent.right) { - index += visibleCount(current.parent.left) + ownVisibleCount(current.parent); - } - current = current.parent; - } - return current === this.#root ? index : undefined; - } - visibleMetricOffsetOf(element, metric) { - if (this.#hasLazyDeletions) - materializeElementDeletion(element, this.#metrics); - const node = elementNode(element); - const elementPhysicalOffset = elementOffset(element); - if (node === undefined || elementPhysicalOffset === undefined) - return undefined; - let offset = visibleMetric(node.left, metric) + - visibleMetricBefore(node, elementPhysicalOffset, metric, this.#metrics); - let current = node; - while (current.parent !== undefined) { - if (current === current.parent.right) { - offset += - visibleMetric(current.parent.left, metric) + - ownVisibleMetric(current.parent, metric, this.#metrics); - } - current = current.parent; - } - return current === this.#root ? offset : undefined; - } - metricOffsetAtVisibleIndex(index, metric) { - if (!Number.isSafeInteger(index) || index < 0 || index > this.visibleLength) { - return undefined; - } - if (index === this.visibleLength) - return visibleMetric(this.#root, metric); - let node = this.#root; - let remaining = index; - let offset = 0; - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - const leftCount = visibleCount(node.left); - if (remaining < leftCount) { - node = node.left; - continue; - } - remaining -= leftCount; - offset += visibleMetric(node.left, metric); - if (remaining < ownVisibleCount(node)) { - const ownOffset = physicalOffsetAtVisibleIndex(node, remaining); - return offset + visibleMetricBefore(node, ownOffset, metric, this.#metrics); - } - remaining -= ownVisibleCount(node); - offset += ownVisibleMetric(node, metric, this.#metrics); - node = node.right; - } - return undefined; - } - visibleIndexAtMetricOffset(offset, metric) { - const total = visibleMetric(this.#root, metric); - if (!Number.isSafeInteger(offset) || offset < 0 || offset > total) - return undefined; - if (offset === total) - return this.visibleLength; - let node = this.#root; - let remaining = offset; - let index = 0; - while (node !== undefined) { - pushNodeDeletion(node, this.#metrics); - const leftMetric = visibleMetric(node.left, metric); - if (remaining < leftMetric) { - node = node.left; - continue; - } - remaining -= leftMetric; - index += visibleCount(node.left); - const ownMetric = ownVisibleMetric(node, metric, this.#metrics); - if (remaining <= ownMetric) { - const ownOffset = physicalOffsetAtMetricOffset(node, remaining, metric); - return ownOffset === undefined - ? undefined - : index + visibleElementsBefore(node, ownOffset); - } - remaining -= ownMetric; - index += ownVisibleCount(node); - node = node.right; - } - return undefined; - } - insertAtVisible(position, elements) { - if (position < 0 || position > this.visibleLength) { - throw new RangeError(`visible sequence position ${position} is out of range`); - } - const physical = position === this.visibleLength - ? this.allLength - : this.physicalIndexOf(this.atVisible(position)); - this.insertAtPhysical(physical, elements); - } - insertSpanAtVisible(position, span) { - if (position < 0 || position > this.visibleLength) { - throw new RangeError(`visible sequence position ${position} is out of range`); - } - const physical = position === this.visibleLength - ? this.allLength - : this.physicalIndexOf(this.atVisible(position)); - this.insertSpanAtPhysical(physical, span); - } - insertAtPhysical(position, elements) { - if (elements.length === 0) - return; - if (position < 0 || position > this.allLength) { - throw new RangeError(`physical sequence position ${position} is out of range`); - } - if (elements.length === 1) { - const id = elements[0].id; - if (this.#locationsByPeer.get(id.peer)?.has(id.counter) === true) { - throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); - } - } - else if (elements.length <= MAX_SEQUENCE_SPAN) { - for (let index = 0; index < elements.length; index += 1) { - const id = elements[index].id; - if (this.#locationsByPeer.get(id.peer)?.has(id.counter) === true) { - throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); - } - for (let previous = 0; previous < index; previous += 1) { - const other = elements[previous].id; - if (other.peer === id.peer && other.counter === id.counter) { - throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); - } - } - } - } - else { - // Same-peer strictly increasing ids cannot duplicate in-batch, so the - // common contiguous batch skips the string-keyed Set. - let idsAreOrdered = true; - let previousPeer = elements[0].id.peer; - let previousCounter = elements[0].id.counter; - for (let index = 1; index < elements.length; index += 1) { - const id = elements[index].id; - if (id.peer !== previousPeer || id.counter <= previousCounter) { - idsAreOrdered = false; - break; - } - previousPeer = id.peer; - previousCounter = id.counter; - } - if (idsAreOrdered) { - for (const element of elements) { - const id = element.id; - if (this.#locationsByPeer.get(id.peer)?.has(id.counter) === true) { - throw new Error(`duplicate sequence id ${id.counter}@${id.peer.toString()}`); - } - } - } - else { - const ids = new Set(); - for (const element of elements) { - const key = `${element.id.peer}:${element.id.counter}`; - if (ids.has(key) || - this.#locationsByPeer.get(element.id.peer)?.has(element.id.counter) === true) { - throw new Error(`duplicate sequence id ${element.id.counter}@${element.id.peer.toString()}`); - } - ids.add(key); - } - } - } - this.#invalidateCausalView(); - if (this.#appendToPreviousNode(position, elements)) { - this.#structureVersion += 1; - return; - } - let insertedRoot; - if (elements.length <= MAX_SEQUENCE_SPAN) { - insertedRoot = this.#newNode(elements, true); - } - else { - for (let start = 0; start < elements.length; start += MAX_SEQUENCE_SPAN) { - insertedRoot = merge(insertedRoot, this.#newNode(elements.slice(start, start + MAX_SEQUENCE_SPAN), true), this.#metrics); - } - } - const appendAtTail = position === this.allLength; - const previousTailElement = appendAtTail || this.#tail === undefined - ? undefined - : nodeElement(this.#tail, nodeLength(this.#tail) - 1); - if (appendAtTail) { - this.#root = this.#appendAtTail(this.#root, insertedRoot); - } - else if (insertedRoot !== undefined && - insertedRoot.left === undefined && - insertedRoot.right === undefined) { - this.#root = this.#insertSingleNode(this.#root, position, insertedRoot); - } - else { - const [left, right] = this.#split(this.#root, position); - this.#root = merge(merge(left, insertedRoot, this.#metrics), right, this.#metrics); - } - if (this.#root !== undefined) - this.#root.parent = undefined; - if (previousTailElement !== undefined) { - this.#tail = elementNode(previousTailElement); - } - this.#structureVersion += 1; - } - insertSpanAtPhysical(position, span) { - if (span.length === 0) - return; - if (position < 0 || position > this.allLength) { - throw new RangeError(`physical sequence position ${position} is out of range`); - } - if (span.length === 1) { - const peer = span.peerAt(0); - const counter = span.counterAt(0); - if (this.#locationsByPeer.get(peer)?.has(counter) === true) { - throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); - } - } - else if (span.idsAreUnique()) { - for (let offset = 0; offset < span.length; offset += 1) { - const peer = span.peerAt(offset); - const counter = span.counterAt(offset); - if (this.#locationsByPeer.get(peer)?.has(counter) === true) { - throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); - } - } - } - else if (span.length <= MAX_SEQUENCE_SPAN) { - for (let offset = 0; offset < span.length; offset += 1) { - const peer = span.peerAt(offset); - const counter = span.counterAt(offset); - if (this.#locationsByPeer.get(peer)?.has(counter) === true) { - throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); - } - for (let previous = 0; previous < offset; previous += 1) { - if (span.peerAt(previous) === peer && span.counterAt(previous) === counter) { - throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); - } - } - } - } - else { - // Same-peer strictly increasing ids cannot duplicate in-batch, so the - // common contiguous span skips the string-keyed Set. - let idsAreOrdered = true; - let previousPeer = span.peerAt(0); - let previousCounter = span.counterAt(0); - for (let offset = 1; offset < span.length; offset += 1) { - const peer = span.peerAt(offset); - const counter = span.counterAt(offset); - if (peer !== previousPeer || counter <= previousCounter) { - idsAreOrdered = false; - break; - } - previousPeer = peer; - previousCounter = counter; - } - if (idsAreOrdered) { - for (let offset = 0; offset < span.length; offset += 1) { - const peer = span.peerAt(offset); - const counter = span.counterAt(offset); - if (this.#locationsByPeer.get(peer)?.has(counter) === true) { - throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); - } - } - } - else { - const ids = new Set(); - for (let offset = 0; offset < span.length; offset += 1) { - const peer = span.peerAt(offset); - const counter = span.counterAt(offset); - const key = `${peer}:${counter}`; - if (ids.has(key) || this.#locationsByPeer.get(peer)?.has(counter) === true) { - throw new Error(`duplicate sequence id ${counter}@${peer.toString()}`); - } - ids.add(key); - } - } - } - this.#invalidateCausalView(); - if (this.#appendSpanToPreviousNode(position, span)) { - this.#structureVersion += 1; - return; - } - let insertedRoot; - if (span.length <= MAX_SEQUENCE_SPAN) { - insertedRoot = this.#newNode(span, true); - } - else { - for (let start = 0; start < span.length; start += MAX_SEQUENCE_SPAN) { - insertedRoot = merge(insertedRoot, this.#newNode(span.slice(start, Math.min(span.length, start + MAX_SEQUENCE_SPAN)), true), this.#metrics); - } - } - const appendAtTail = position === this.allLength; - const previousTailElement = appendAtTail || this.#tail === undefined - ? undefined - : nodeElement(this.#tail, nodeLength(this.#tail) - 1); - if (appendAtTail) { - this.#root = this.#appendAtTail(this.#root, insertedRoot); - } - else if (insertedRoot !== undefined && - insertedRoot.left === undefined && - insertedRoot.right === undefined) { - this.#root = this.#insertSingleNode(this.#root, position, insertedRoot); - } - else { - const [left, right] = this.#split(this.#root, position); - this.#root = merge(merge(left, insertedRoot, this.#metrics), right, this.#metrics); - } - if (this.#root !== undefined) - this.#root.parent = undefined; - if (previousTailElement !== undefined) { - this.#tail = elementNode(previousTailElement); - } - this.#structureVersion += 1; - } - setDeleted(element, deleted) { - if (this.#hasLazyDeletions) - materializeElementDeletion(element, this.#metrics); - const node = elementNode(element); - const offset = elementOffset(element); - if (node === undefined || offset === undefined || element.deleted === deleted) - return; - this.#invalidateCausalView(); - updateNodeElementVisibility(node, offset, element, deleted, this.#metrics); - recomputeToRoot(node, this.#metrics, false); - } - deleteElement(element, deletedBy) { - if (this.#hasLazyDeletions) - materializeElementDeletion(element, this.#metrics); - const node = elementNode(element); - if (node === undefined) { - throw new Error("cannot delete an element outside the sequence"); - } - if (element.deleted && deletedBy === undefined) - return; - this.#invalidateCausalView(); - if (!element.deleted) { - updateNodeElementVisibility(node, elementOffset(element), element, true, this.#metrics); - recomputeToRoot(node, this.#metrics, false); - } - if (deletedBy === undefined) - return; - if (element.deletedByPeer === undefined && (element.deletedBy?.length ?? 0) === 0) { - element.deletedByPeer = deletedBy.peer; - element.deletedByCounter = deletedBy.counter; - } - else - (element.deletedBy ??= []).push(deletedBy); - this.#recordDeletionElement(element, deletedBy); - const previous = this.#maxCounterByPeer.get(deletedBy.peer) ?? 0; - if (deletedBy.counter + 1 > previous) { - this.#maxCounterByPeer.set(deletedBy.peer, deletedBy.counter + 1); - } - } - recordOperationId(id) { - const previous = this.#maxCounterByPeer.get(id.peer) ?? 0; - const next = Math.max(previous, id.counter + 1); - if (next === previous) - return; - this.#invalidateCausalView(); - this.#maxCounterByPeer.set(id.peer, next); - } - addDeletion(element, id) { - if (elementNode(element) === undefined) { - throw new Error("cannot delete an element outside the sequence"); - } - this.#invalidateCausalView(); - if (element.deletedByPeer === undefined && (element.deletedBy?.length ?? 0) === 0) { - element.deletedByPeer = id.peer; - element.deletedByCounter = id.counter; - } - else - (element.deletedBy ??= []).push(id); - this.#recordDeletionElement(element, id); - this.recordOperationId(id); - } - deleteIdSpan(startId, length, deletedBy) { - const spanLength = Math.abs(length); - if (spanLength === 0) - return; - if (spanLength === 1) { - const element = this.findById(startId); - if (element === undefined) - return; - this.deleteElement(element, deletedBy); - return; - } - const targetedLimit = Math.max(64, Math.min(512, this.allLength >>> 8)); - const deleteByTree = this.#root?.idRunCount === 1 || spanLength > targetedLimit; - const result = deleteByTree - ? markIdRunsDeleted(this.#root, singleCounterRangeRelation(startId.peer, startId.counter, startId.counter + spanLength), this.#metrics) - : this.#markLocatedIdSpanVisibility(startId, spanLength, true); - if (!result.found) - return; - if (deleteByTree) - this.#hasLazyDeletions = true; - this.#invalidateCausalView(); - if (deletedBy !== undefined) { - this.#deletionRuns.add(startId, spanLength, deletedBy, length >= 0 ? 1 : -1); - this.recordOperationId({ - peer: deletedBy.peer, - counter: deletedBy.counter + spanLength - 1, - }); - } - } - #markLocatedIdSpanVisibility(startId, length, deleted) { - const locations = this.#locationsByPeer.get(startId.peer); - if (locations === undefined) - return { found: false, changed: false }; - // Resolves locations to (node, offset) pairs so compact-span targets are - // read and written through column helpers without materializing views. - const targetNodes = []; - const targetOffsets = []; - locations.forEach(startId.counter, startId.counter + length, (location) => { - if (typeof location === "number") { - targetNodes.push(this.#nodeAtLocation(location)); - targetOffsets.push(location % SEQUENCE_LOCATION_STRIDE); - } - else { - targetNodes.push(elementNode(location)); - targetOffsets.push(elementOffset(location)); - } - }); - if (targetNodes.length === 0) - return { found: false, changed: false }; - if (targetNodes.length <= MAX_SEQUENCE_SPAN) { - const materializedNodes = []; - for (const node of targetNodes) { - if (materializedNodes.includes(node)) - continue; - materializedNodes.push(node); - materializeNodeDeletion(node, this.#metrics); - } - const changedNodes = []; - for (let index = 0; index < targetNodes.length; index += 1) { - const node = targetNodes[index]; - if (nodeDeleted(node, targetOffsets[index]) === deleted) - continue; - setNodeDeleted(node, targetOffsets[index], deleted); - if (!changedNodes.includes(node)) - changedNodes.push(node); - } - if (changedNodes.length === 0) - return { found: true, changed: false }; - for (const node of changedNodes) - recomputeOwn(node, this.#metrics); - for (const node of changedNodes) - recomputeToRoot(node, this.#metrics, false); - return { found: true, changed: true }; - } - const materializedNodes = new Set(); - for (const node of targetNodes) { - if (materializedNodes.has(node)) - continue; - materializedNodes.add(node); - materializeNodeDeletion(node, this.#metrics); - } - const changedNodes = new Set(); - for (let index = 0; index < targetNodes.length; index += 1) { - const node = targetNodes[index]; - if (nodeDeleted(node, targetOffsets[index]) === deleted) - continue; - setNodeDeleted(node, targetOffsets[index], deleted); - changedNodes.add(node); - } - if (changedNodes.size === 0) - return { found: true, changed: false }; - const affected = new Set(); - for (const node of changedNodes) { - recomputeOwn(node, this.#metrics); - let current = node; - while (current !== undefined) { - affected.add(current); - current = current.parent; - } - } - recomputeAffected(this.#root, affected, this.#metrics); - return { found: true, changed: true }; - } - setIdRunsDeleted(runs) { - if (runs.length === 0) - return; - const run = runs.length === 1 ? runs[0] : undefined; - const targetedLimit = Math.max(64, Math.min(512, this.allLength >>> 8)); - const updateByTree = run === undefined || this.#root?.idRunCount === 1 || run.length > targetedLimit; - const result = updateByTree - ? markIdRunsDeleted(this.#root, run === undefined - ? mapCounterRangeRelation(counterRangesByPeer(runs)) - : singleCounterRangeRelation(run.start.peer, run.start.counter, run.start.counter + run.length), this.#metrics) - : this.#markLocatedIdSpanVisibility(run.start, run.length, true); - if (!result.changed) - return; - if (updateByTree) - this.#hasLazyDeletions = true; - this.#invalidateCausalView(); - } - setIdRunsVisible(runs) { - if (runs.length === 0) - return; - const run = runs.length === 1 ? runs[0] : undefined; - const targetedLimit = Math.max(64, Math.min(512, this.allLength >>> 8)); - const updateByTree = run === undefined || this.#root?.idRunCount === 1 || run.length > targetedLimit; - const result = updateByTree - ? markIdRunsVisible(this.#root, run === undefined - ? mapCounterRangeRelation(counterRangesByPeer(runs)) - : singleCounterRangeRelation(run.start.peer, run.start.counter, run.start.counter + run.length), this.#metrics) - : this.#markLocatedIdSpanVisibility(run.start, run.length, false); - if (!result.changed) - return; - if (updateByTree) - this.#hasLazyDeletions = true; - this.#invalidateCausalView(); - } - canShowIdRunsAt(runs, version) { - for (const run of runs) { - if (run.start.counter + run.length > (version.get(run.start.peer) ?? 0)) { - return false; - } - } - if (this.#deletionRuns.hasDeletionAt(runs, version)) - return false; - for (const run of runs) { - if (this.#scalarDeletionTargetsByPeer - .get(run.start.peer) - ?.some(run.start.counter, run.start.counter + run.length, (location) => someSequenceDeletion(this.#elementAtDeletionLocation(location), (id) => id.counter < (version.get(id.peer) ?? 0)))) { - return false; - } - } - return true; - } - elementsDeletedBy(peer, start, end) { - const selected = new Set(); - for (const run of this.#deletionRuns.targetRunsDeletedBy(peer, start, end)) { - for (let offset = 0; offset < run.length; offset += 1) { - const location = this.#locationsByPeer - .get(run.start.peer) - ?.get(run.start.counter + offset); - if (location !== undefined) { - selected.add(typeof location === "number" - ? { - stableSequenceId: true, - peer: run.start.peer, - counter: run.start.counter + offset, - } - : location); - } - } - } - this.#deletionsByPeer.get(peer)?.forEach(start, end, (elementOrSet) => { - if (elementOrSet instanceof Set) { - for (const element of elementOrSet) - selected.add(element); - } - else { - selected.add(elementOrSet); - } - }); - const resolved = []; - for (const location of selected) { - const element = this.#elementAtDeletionLocation(location); - resolved.push({ element, index: this.physicalIndexOf(element) }); - } - resolved.sort((left, right) => left.index - right.index); - const output = []; - for (const { element } of resolved) - output.push(element); - return output; - } - idRunsDeletedBy(peer, start, end) { - const runs = this.#deletionRuns.targetRunsDeletedBy(peer, start, end); - this.#deletionsByPeer.get(peer)?.forEach(start, end, (elementOrSet) => { - if (elementOrSet instanceof Set) { - for (const location of elementOrSet) { - runs.push({ start: this.#elementAtDeletionLocation(location).id, length: 1 }); - } - } - else { - runs.push({ - start: this.#elementAtDeletionLocation(elementOrSet).id, - length: 1, - }); - } - }); - return normalizeSequenceIdRuns(runs); - } - someDeletion(element, predicate) { - return (this.#deletionRuns.someDeletion(element.id, predicate) || - someSequenceDeletion(element, predicate)); - } - everyDeletion(element, predicate) { - return (this.#deletionRuns.everyDeletion(element.id, predicate) && - everySequenceDeletion(element, predicate)); - } - moveVisible(from, to) { - const element = this.atVisible(from); - if (element === undefined || from === to) - return; - this.#invalidateCausalView(); - const physical = this.physicalIndexOf(element); - const [before, selectedAndAfter] = this.#split(this.#root, physical); - const [selected, after] = this.#split(selectedAndAfter, 1); - this.#root = merge(before, after, this.#metrics); - const destinationElement = this.atVisible(to); - const destination = destinationElement === undefined - ? this.allLength - : this.physicalIndexOf(destinationElement); - const [left, right] = this.#split(this.#root, destination); - this.#root = merge(merge(left, selected, this.#metrics), right, this.#metrics); - if (this.#root !== undefined) - this.#root.parent = undefined; - this.#tail = this.#root === undefined ? undefined : rightmost(this.#root); - this.#structureVersion += 1; - } - moveBefore(element, before) { - if (element === before) - return; - const physical = this.physicalIndexOf(element); - if (physical === undefined) - return; - this.#invalidateCausalView(); - const [left, selectedAndRight] = this.#split(this.#root, physical); - const [selected, right] = this.#split(selectedAndRight, 1); - this.#root = merge(left, right, this.#metrics); - const destination = before === undefined ? this.allLength : this.physicalIndexOf(before); - if (destination === undefined) { - this.#root = merge(this.#root, selected, this.#metrics); - } - else { - const [beforeDestination, afterDestination] = this.#split(this.#root, destination); - this.#root = merge(merge(beforeDestination, selected, this.#metrics), afterDestination, this.#metrics); - } - if (this.#root !== undefined) - this.#root.parent = undefined; - this.#tail = this.#root === undefined ? undefined : rightmost(this.#root); - this.#structureVersion += 1; - } - nextVisible(element) { - if (this.#hasLazyDeletions) - materializeElementDeletion(element, this.#metrics); - const node = elementNode(element); - const offset = elementOffset(element); - if (node === undefined || offset === undefined) - return undefined; - for (let index = offset + 1; index < nodeLength(node); index += 1) { - if (!nodeDeleted(node, index)) - return nodeElement(node, index); - } - const right = firstVisibleElement(node.right, this.#metrics); - if (right !== undefined) - return right; - let current = node; - while (current.parent !== undefined) { - const parent = current.parent; - if (current === parent.left) { - for (let index = 0; index < nodeLength(parent); index += 1) { - if (!nodeDeleted(parent, index)) - return nodeElement(parent, index); - } - const sibling = firstVisibleElement(parent.right, this.#metrics); - if (sibling !== undefined) - return sibling; - } - current = parent; - } - return undefined; - } - previousVisible(element) { - if (this.#hasLazyDeletions) - materializeElementDeletion(element, this.#metrics); - const node = elementNode(element); - const offset = elementOffset(element); - if (node === undefined || offset === undefined) - return undefined; - for (let index = offset - 1; index >= 0; index -= 1) { - if (!nodeDeleted(node, index)) - return nodeElement(node, index); - } - const left = lastVisibleElement(node.left, this.#metrics); - if (left !== undefined) - return left; - let current = node; - while (current.parent !== undefined) { - const parent = current.parent; - if (current === parent.right) { - for (let index = nodeLength(parent) - 1; index >= 0; index -= 1) { - if (!nodeDeleted(parent, index)) - return nodeElement(parent, index); - } - const sibling = lastVisibleElement(parent.left, this.#metrics); - if (sibling !== undefined) - return sibling; - } - current = parent; - } - return undefined; - } - isFullyIncluded(version) { - for (const [peer, counter] of this.#maxCounterByPeer) { - if ((version.get(peer) ?? 0) < counter) - return false; - } - return true; - } - causalVisible(version) { - if (this.isFullyIncluded(version)) - return this.visible(); - const view = this.causalView(version); - return view.range(0, view.length); - } - causalView(version) { - if (this.isFullyIncluded(version)) { - return { - length: this.visibleLength, - at: (index) => this.atVisible(index), - range: (start, end) => this.visibleRange(start, end), - idRuns: (start, end) => this.visibleIdRuns(start, end), - }; - } - const cachedIndex = this.#cachedCausalViews.findIndex((cached) => versionsEqual(cached.version, version)); - if (cachedIndex >= 0) { - const cached = this.#cachedCausalViews[cachedIndex]; - if (cachedIndex + 1 < this.#cachedCausalViews.length) { - this.#cachedCausalViews.splice(cachedIndex, 1); - this.#cachedCausalViews.push(cached); - } - return cached.view; - } - const causalVersion = new Map(version); - const included = (id) => id.counter < (causalVersion.get(id.peer) ?? 0); - const deletionIncluded = (deleteId) => !included(deleteId); - // Column-read visibility probe: scans stay allocation-free over compact - // spans and only returned elements are materialized by the callers. - const isVisible = (node, offset) => { - scratchSequenceCausalId.peer = nodePeer(node, offset); - scratchSequenceCausalId.counter = nodeCounter(node, offset); - return (included(scratchSequenceCausalId) && - this.#deletionRuns.everyDeletion(scratchSequenceCausalId, deletionIncluded) && - everyNodeElementDeletion(node, offset, deletionIncluded)); - }; - const causalCounts = new Map(); - const causalCount = (node) => { - if (node === undefined) - return 0; - const cached = causalCounts.get(node); - if (cached !== undefined) - return cached; - if (node.idRunCount === 1) { - const includedEnd = causalVersion.get(node.firstId.peer) ?? 0; - if (includedEnd <= node.firstId.counter) { - causalCounts.set(node, 0); - return 0; - } - if (includedEnd > node.lastId.counter && - this.canShowIdRunsAt([{ start: node.firstId, length: node.allCount }], causalVersion)) { - causalCounts.set(node, node.allCount); - return node.allCount; - } - } - let count = causalCount(node.left) + causalCount(node.right); - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (isVisible(node, offset)) - count += 1; - } - causalCounts.set(node, count); - return count; - }; - const length = causalCount(this.#root); - const view = { - length, - at: (index) => { - if (!Number.isSafeInteger(index) || index < 0 || index >= length) { - return undefined; - } - let node = this.#root; - let remaining = index; - while (node !== undefined) { - const leftCount = causalCount(node.left); - if (remaining < leftCount) { - node = node.left; - continue; - } - remaining -= leftCount; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (!isVisible(node, offset)) - continue; - if (remaining === 0) - return nodeElement(node, offset); - remaining -= 1; - } - node = node.right; - } - return undefined; - }, - range: (start, end) => { - const boundedStart = Math.max(0, Math.min(start, length)); - const boundedEnd = Math.max(boundedStart, Math.min(end, length)); - const output = []; - collectCausalRange(this.#root, boundedStart, boundedEnd, output, causalCount, isVisible, this.#metrics); - return output; - }, - idRuns: (start, end) => { - const boundedStart = Math.max(0, Math.min(start, length)); - const boundedEnd = Math.max(boundedStart, Math.min(end, length)); - const output = []; - collectCausalIdRuns(this.#root, boundedStart, boundedEnd, output, causalCount, isVisible); - return output; - }, - }; - this.#cachedCausalViews.push({ version: causalVersion, view }); - if (this.#cachedCausalViews.length > MAX_CACHED_CAUSAL_VIEWS) { - this.#cachedCausalViews.shift(); - } - return view; - } - #invalidateCausalView() { - this.#cachedCausalViews.length = 0; - } - reset() { - this.#invalidateCausalView(); - this.#root = undefined; - this.#tail = undefined; - this.#locationsByPeer.clear(); - this.#locationsByLamportPeer.clear(); - this.#nodesByLocationId.length = 0; - this.#deletionsByPeer.clear(); - this.#scalarDeletionTargetsByPeer.clear(); - this.#deletionRuns.reset(); - this.#maxCounterByPeer.clear(); - this.#hasLazyDeletions = false; - this.#structureVersion += 1; - } - reservePeerCounters(peer, endExclusive) { - if (!Number.isSafeInteger(endExclusive) || endExclusive < 0) { - throw new RangeError("sequence peer counter capacity is out of range"); - } - let locations = this.#locationsByPeer.get(peer); - if (locations === undefined) { - locations = new CounterStore(); - this.#locationsByPeer.set(peer, locations); - } - locations.reserveDense(endExclusive); - } - /** - * Loads globally validated spans into an empty index in physical order. - * The Cartesian-tree build preserves the same priority treap as repeated - * tail insertion while recomputing every node only once. - */ - loadValidatedSpans(spans) { - if (this.#root !== undefined || this.#nodesByLocationId.length !== 0) { - throw new Error("validated sequence spans require an empty index"); - } - if (spans.length === 0) - return; - this.#invalidateCausalView(); - const stack = []; - let root; - let tail; - for (const span of spans) { - const node = this.#newNode(span, true); - tail = node; - let left; - while (stack.length > 0 && stack[stack.length - 1].priority > node.priority) { - left = stack.pop(); - } - node.left = left; - const parent = stack[stack.length - 1]; - if (parent === undefined) { - root = node; - } - else { - parent.right = node; - } - stack.push(node); - } - const pending = [root]; - const postorder = []; - while (pending.length > 0) { - const node = pending.pop(); - postorder.push(node); - if (node.left !== undefined) - pending.push(node.left); - if (node.right !== undefined) - pending.push(node.right); - } - for (let index = postorder.length - 1; index >= 0; index -= 1) { - recompute(postorder[index], this.#metrics); - } - root.parent = undefined; - this.#root = root; - this.#tail = tail; - this.#structureVersion += 1; - } - #newNode(storage, recordNew) { - const element = Array.isArray(storage) - ? storage.length === 1 - ? storage[0] - : [...storage] - : storage; - const isSpan = element instanceof SequenceSpan; - const length = isSpan - ? element.length - : Array.isArray(element) - ? element.length - : 1; - if (length === 0 || length > MAX_SEQUENCE_SPAN) { - throw new RangeError("sequence span length is out of range"); - } - this.#randomState ^= this.#randomState << 13; - this.#randomState ^= this.#randomState >>> 17; - this.#randomState ^= this.#randomState << 5; - // Keep priorities inside the compressed-Smi range [0, 2^30) so every node - // avoids boxing its priority; the treap only needs a deterministic total - // order, and 30 bits of entropy is ample for its node counts. - const priority = this.#randomState >>> 2; - const locationId = this.#nodesByLocationId.length; - const node = { - element, - isSpan, - ownCount: length, - locationId, - priority, - left: undefined, - right: undefined, - parent: undefined, - ownVisibleCount: 0, - ownVisibleUtf16: 0, - ownVisibleUtf8: 0, - ownFirstVisibleId: undefined, - ownLastVisibleId: undefined, - ownVisibleIdRunCount: 0, - ownFirstId: { peer: 0n, counter: 0 }, - ownLastId: { peer: 0n, counter: 0 }, - ownIdRunCount: 0, - ownUtf16: 0, - ownUtf8: 0, - visibleOffsets: undefined, - visibleUtf16Prefix: undefined, - visibleUtf8Prefix: undefined, - allCount: 0, - visibleCount: 0, - visibleUtf16: 0, - visibleUtf8: 0, - firstVisibleId: undefined, - lastVisibleId: undefined, - visibleIdRunCount: 0, - firstId: { peer: 0n, counter: 0 }, - lastId: { peer: 0n, counter: 0 }, - idRunCount: 0, - allUtf16: 0, - allUtf8: 0, - lazyDeleted: false, - lazyVisible: false, - }; - this.#nodesByLocationId.push(node); - recomputeOwn(node, this.#metrics); - recompute(node, this.#metrics); - if (recordNew) - this.#recordNewElements(node); - else - this.#assignLocations(node); - return node; - } - #recordNewElements(node) { - for (let offset = 0; offset < nodeLength(node); offset += 1) { - this.#recordNewElement(node, offset); - } - } - #recordNewElement(node, offset) { - if (!node.isSpan) { - this.#recordNewPlainElement(node, offset, nodeElement(node, offset)); - return; - } - const idPeer = nodePeer(node, offset); - const idCounter = nodeCounter(node, offset); - const location = node.locationId * SEQUENCE_LOCATION_STRIDE + offset; - let peer = this.#locationsByPeer.get(idPeer); - if (peer === undefined) { - peer = new CounterStore(); - this.#locationsByPeer.set(idPeer, peer); - } - peer.set(idCounter, location); - const lamport = nodeLamport(node, offset); - if (lamport !== undefined) { - this.#locationsByLamportPeer.get(idPeer)?.set(lamport, location); - } - const operationEnd = idCounter + 1; - const previousOperationEnd = this.#maxCounterByPeer.get(idPeer) ?? 0; - if (operationEnd > previousOperationEnd) { - this.#maxCounterByPeer.set(idPeer, operationEnd); - } - const deletionIds = node.element.deletionIdsAt(offset); - if (deletionIds.length > 0) { - const element = nodeElement(node, offset); - for (const deletionId of deletionIds) { - this.#recordDeletionElement(element, deletionId); - this.recordOperationId(deletionId); - } - } - } - #recordNewPlainElement(node, offset, element) { - setElementLocation(element, node, offset); - let peer = this.#locationsByPeer.get(element.id.peer); - if (peer === undefined) { - peer = new CounterStore(); - this.#locationsByPeer.set(element.id.peer, peer); - } - peer.set(element.id.counter, element); - if (element.lamport !== undefined) { - this.#locationsByLamportPeer.get(element.id.peer)?.set(element.lamport, element); - } - const operationEnd = element.id.counter + 1; - const previousOperationEnd = this.#maxCounterByPeer.get(element.id.peer) ?? 0; - if (operationEnd > previousOperationEnd) { - this.#maxCounterByPeer.set(element.id.peer, operationEnd); - } - // Iterates deletion metadata in place: the common fresh element carries - // none and must not allocate an empty id list. - if (element.deletedByPeer !== undefined && element.deletedByCounter !== undefined) { - const deletionId = { - peer: element.deletedByPeer, - counter: element.deletedByCounter, - }; - this.#recordDeletionElement(element, deletionId); - this.recordOperationId(deletionId); - } - const deletedBy = element.deletedBy; - if (deletedBy !== undefined) { - for (let index = 0; index < deletedBy.length; index += 1) { - const deletionId = deletedBy[index]; - this.#recordDeletionElement(element, deletionId); - this.recordOperationId(deletionId); - } - } - } - #assignLocations(node) { - if (node.isSpan) { - node.element.forEachRetained((element, offset) => { - setElementLocation(element, node, offset); - }); - } - else { - for (let offset = 0; offset < nodeLength(node); offset += 1) { - const element = nodeElement(node, offset); - if (elementNode(element) === undefined) { - throw new Error("cannot relocate an unindexed sequence element"); - } - setElementLocation(element, node, offset); - } - } - for (let offset = 0; offset < nodeLength(node); offset += 1) { - const peer = nodePeer(node, offset); - const counter = nodeCounter(node, offset); - const location = node.isSpan - ? node.locationId * SEQUENCE_LOCATION_STRIDE + offset - : nodeElement(node, offset); - this.#locationsByPeer.get(peer)?.set(counter, location); - const lamport = nodeLamport(node, offset); - if (lamport !== undefined) { - this.#locationsByLamportPeer.get(peer)?.set(lamport, location); - } - } - } - #nodeAtLocation(location) { - // location is non-negative and below 2^31; SEQUENCE_LOCATION_STRIDE is 64. - const node = this.#nodesByLocationId[(location / SEQUENCE_LOCATION_STRIDE) | 0]; - if (node === undefined) - throw new Error("sequence location points to a missing node"); - return node; - } - #elementAtLocation(location) { - return nodeElement(this.#nodeAtLocation(location), location % SEQUENCE_LOCATION_STRIDE); - } - #elementAtDeletionLocation(location) { - if ("stableSequenceId" in location) { - const element = this.findById(location); - if (element === undefined) - throw new Error("deleted sequence element is missing"); - return element; - } - return location; - } - #recordDeletionElement(element, id) { - const node = elementNode(element); - const offset = elementOffset(element); - const location = node?.isSpan === true && offset !== undefined - ? { - stableSequenceId: true, - peer: element.id.peer, - counter: element.id.counter, - } - : element; - let targets = this.#scalarDeletionTargetsByPeer.get(element.id.peer); - if (targets === undefined) { - targets = new PagedCounterStore(); - this.#scalarDeletionTargetsByPeer.set(element.id.peer, targets); - } - targets.set(element.id.counter, location); - let counters = this.#deletionsByPeer.get(id.peer); - if (counters === undefined) { - counters = new CounterStore(); - this.#deletionsByPeer.set(id.peer, counters); - } - const elementOrSet = counters.get(id.counter); - if (elementOrSet === undefined) { - counters.set(id.counter, location); - } - else if (elementOrSet instanceof Set) { - elementOrSet.add(location); - } - else if (elementOrSet !== location) { - counters.set(id.counter, new Set([elementOrSet, location])); - } - } - #split(root, count) { - if (root === undefined) - return [undefined, undefined]; - // Iterative split: descend to the cut node, then stitch the two result - // trees on the way back up through the original parent links. This - // mirrors the recursive unwind without a per-level [left, right] tuple. - let cut = root; - let last; - let lastWentLeft = false; - let remaining = count; - let cutLeftCount = 0; - let cutOwnEnd = 0; - while (cut !== undefined) { - last = cut; - pushNodeDeletion(cut, this.#metrics); - const leftCount = allCount(cut.left); - const ownEnd = leftCount + nodeLength(cut); - if (remaining < leftCount) { - lastWentLeft = true; - cut = cut.left; - } - else if (remaining > ownEnd) { - lastWentLeft = false; - remaining -= ownEnd; - cut = cut.right; - } - else { - cutLeftCount = leftCount; - cutOwnEnd = ownEnd; - break; - } - } - let leftResult; - let rightResult; - let child; - const cutParent = (cut !== undefined ? cut : last).parent; - if (cut !== undefined) { - if (remaining === cutLeftCount) { - leftResult = cut.left; - cut.left = undefined; - recompute(cut, this.#metrics); - cut.parent = undefined; - if (leftResult !== undefined) - leftResult.parent = undefined; - rightResult = cut; - } - else if (remaining === cutOwnEnd) { - rightResult = cut.right; - cut.right = undefined; - recompute(cut, this.#metrics); - cut.parent = undefined; - if (rightResult !== undefined) - rightResult.parent = undefined; - leftResult = cut; - } - else { - const ownOffset = remaining - cutLeftCount; - const storage = cut.element; - const oldRight = cut.right; - cut.element = sliceNodeStorage(storage, 0, ownOffset); - cut.ownCount = ownOffset; - cut.right = undefined; - recomputeOwn(cut, this.#metrics); - recompute(cut, this.#metrics); - cut.parent = undefined; - this.#assignLocations(cut); - const rightRoot = this.#newNode(sliceNodeStorage(storage, ownOffset, nodeLengthFromStorage(storage)), false); - rightResult = merge(rightRoot, oldRight, this.#metrics); - if (rightResult !== undefined) - rightResult.parent = undefined; - leftResult = cut; - } - child = cut; - } - else { - // `count` fell outside the tree: the deepest frame reattaches nothing - // and folds the last visited node into the matching result. - child = last; - if (lastWentLeft) { - child.left = undefined; - recompute(child, this.#metrics); - child.parent = undefined; - rightResult = child; - } - else { - child.right = undefined; - recompute(child, this.#metrics); - child.parent = undefined; - leftResult = child; - } - } - let up = cutParent; - while (child !== root && up !== undefined) { - const nextUp = up.parent; - if (up.left === child) { - up.left = rightResult; - recompute(up, this.#metrics); - if (leftResult !== undefined) - leftResult.parent = undefined; - up.parent = undefined; - rightResult = up; - } - else { - up.right = leftResult; - recompute(up, this.#metrics); - if (rightResult !== undefined) - rightResult.parent = undefined; - up.parent = undefined; - leftResult = up; - } - child = up; - up = nextUp; - } - return [leftResult, rightResult]; - } - #appendToPreviousNode(position, elements) { - if (position === 0) - return false; - const previous = this.atPhysical(position - 1); - if (previous === undefined) - return false; - const node = elementNode(previous); - const offset = elementOffset(previous); - const length = nodeLength(node); - if (node.isSpan || - offset !== length - 1 || - length + elements.length > MAX_SEQUENCE_SPAN) { - return false; - } - const appendingAtTail = position === this.allLength; - const storage = node.element; - const values = Array.isArray(storage) ? storage : [storage]; - node.element = values; - const visibleOffsets = node.visibleOffsets ?? [0, node.ownVisibleCount]; - const visibleUtf16Prefix = node.visibleUtf16Prefix ?? [0, node.ownVisibleUtf16]; - const visibleUtf8Prefix = node.visibleUtf8Prefix ?? [0, node.ownVisibleUtf8]; - for (let offset = 0; offset < elements.length; offset += 1) { - const element = elements[offset]; - values.push(element); - this.#recordNewElement(node, length + offset); - if (!sequenceIdsContinue(node.ownLastId, element.id)) { - node.ownIdRunCount += 1; - } - node.ownLastId = element.id; - const metrics = this.#metrics(element); - node.ownUtf16 += metrics.utf16; - node.ownUtf8 += metrics.utf8; - if (!element.deleted) { - if (node.ownVisibleCount === 0) { - node.ownFirstVisibleId = element.id; - node.ownVisibleIdRunCount = 1; - } - else if (!sequenceIdsContinue(node.ownLastVisibleId, element.id)) { - node.ownVisibleIdRunCount += 1; - } - node.ownLastVisibleId = element.id; - node.ownVisibleCount += 1; - node.ownVisibleUtf16 += metrics.utf16; - node.ownVisibleUtf8 += metrics.utf8; - } - visibleOffsets.push(node.ownVisibleCount); - visibleUtf16Prefix.push(node.ownVisibleUtf16); - visibleUtf8Prefix.push(node.ownVisibleUtf8); - } - node.ownCount += elements.length; - node.visibleOffsets = visibleOffsets; - node.visibleUtf16Prefix = visibleUtf16Prefix; - node.visibleUtf8Prefix = visibleUtf8Prefix; - recomputeToRoot(node, this.#metrics); - if (appendingAtTail) - this.#tail = node; - return true; - } - #appendSpanToPreviousNode(position, span) { - if (position === 0) - return false; - const previous = this.atPhysical(position - 1); - if (previous === undefined) - return false; - const node = elementNode(previous); - const offset = elementOffset(previous); - const length = nodeLength(node); - if (!node.isSpan || - offset !== length - 1 || - length + span.length > MAX_SEQUENCE_SPAN || - !node.element.append(span)) { - return false; - } - node.ownCount += span.length; - const appendingAtTail = position === this.allLength; - const visibleOffsets = node.visibleOffsets ?? [0, node.ownVisibleCount]; - const visibleUtf16Prefix = node.visibleUtf16Prefix ?? [0, node.ownVisibleUtf16]; - const visibleUtf8Prefix = node.visibleUtf8Prefix ?? [0, node.ownVisibleUtf8]; - let lastPeer = node.ownLastId.peer; - let lastCounter = node.ownLastId.counter; - let lastVisiblePeer = node.ownLastVisibleId?.peer; - let lastVisibleCounter = node.ownLastVisibleId?.counter ?? 0; - for (let insertedOffset = 0; insertedOffset < span.length; insertedOffset += 1) { - const offsetInNode = length + insertedOffset; - this.#recordNewElement(node, offsetInNode); - const peer = nodePeer(node, offsetInNode); - const counter = nodeCounter(node, offsetInNode); - if (!sequenceIdScalarsContinue(lastPeer, lastCounter, peer, counter)) { - node.ownIdRunCount += 1; - } - lastPeer = peer; - lastCounter = counter; - const value = nodeMetrics(node, offsetInNode, this.#metrics); - node.ownUtf16 += value.utf16; - node.ownUtf8 += value.utf8; - if (!nodeDeleted(node, offsetInNode)) { - if (node.ownVisibleCount === 0) { - node.ownFirstVisibleId = { peer, counter }; - node.ownVisibleIdRunCount = 1; - } - else if (!sequenceIdScalarsContinue(lastVisiblePeer, lastVisibleCounter, peer, counter)) { - node.ownVisibleIdRunCount += 1; - } - lastVisiblePeer = peer; - lastVisibleCounter = counter; - node.ownVisibleCount += 1; - node.ownVisibleUtf16 += value.utf16; - node.ownVisibleUtf8 += value.utf8; - } - visibleOffsets.push(node.ownVisibleCount); - visibleUtf16Prefix.push(node.ownVisibleUtf16); - visibleUtf8Prefix.push(node.ownVisibleUtf8); - } - node.ownLastId = { peer: lastPeer, counter: lastCounter }; - node.ownLastVisibleId = - lastVisiblePeer === undefined - ? undefined - : { peer: lastVisiblePeer, counter: lastVisibleCounter }; - node.visibleOffsets = visibleOffsets; - node.visibleUtf16Prefix = visibleUtf16Prefix; - node.visibleUtf8Prefix = visibleUtf8Prefix; - recomputeToRoot(node, this.#metrics); - if (appendingAtTail) - this.#tail = node; - return true; - } - #insertSingleNode(root, position, inserted) { - if (root === undefined) { - inserted.parent = undefined; - return inserted; - } - pushNodeDeletion(root, this.#metrics); - if (inserted.priority < root.priority) { - const [left, right] = this.#split(root, position); - inserted.left = left; - inserted.right = right; - recompute(inserted, this.#metrics); - inserted.parent = undefined; - return inserted; - } - const leftCount = allCount(root.left); - const ownEnd = leftCount + nodeLength(root); - if (position <= leftCount) { - root.left = this.#insertSingleNode(root.left, position, inserted); - recompute(root, this.#metrics); - root.parent = undefined; - return root; - } - if (position >= ownEnd) { - root.right = this.#insertSingleNode(root.right, position - ownEnd, inserted); - recompute(root, this.#metrics); - root.parent = undefined; - return root; - } - const [left, right] = this.#split(root, position); - return merge(merge(left, inserted, this.#metrics), right, this.#metrics); - } - #appendAtTail(left, right) { - if (left === undefined || right === undefined) { - if (right !== undefined) - this.#tail = rightmost(right); - return merge(left, right, this.#metrics); - } - this.#tail = rightmost(right); - return merge(left, right, this.#metrics); - } -} -// Reused deletion id passed to predicates so each visited deleted element -// does not allocate a short-lived id object. Predicates must not retain the -// id argument or query deletions reentrantly. -const scratchSequenceDeletionId = { peer: 0n, counter: 0 }; -// Reused id targets for visible-id-run starts and causal-view visibility -// probes, so column scans do not allocate an id per visited element. -// `appendSequenceIdRun` copies the run start before retaining it; deletion -// predicates must not retain the ids they receive. -const scratchSequenceRunId = { peer: 0n, counter: 0 }; -const scratchSequenceCausalId = { peer: 0n, counter: 0 }; -export function someSequenceDeletion(element, predicate) { - if (element.deletedByPeer !== undefined && element.deletedByCounter !== undefined) { - scratchSequenceDeletionId.peer = element.deletedByPeer; - scratchSequenceDeletionId.counter = element.deletedByCounter; - if (predicate(scratchSequenceDeletionId)) - return true; - } - return element.deletedBy?.some(predicate) ?? false; -} -export function everySequenceDeletion(element, predicate) { - if (element.deletedByPeer !== undefined && element.deletedByCounter !== undefined) { - scratchSequenceDeletionId.peer = element.deletedByPeer; - scratchSequenceDeletionId.counter = element.deletedByCounter; - if (!predicate(scratchSequenceDeletionId)) - return false; - } - return element.deletedBy?.every(predicate) ?? true; -} -/** - * Column-read variant of `everySequenceDeletion`: compact-span deletion - * metadata is checked through `deletionIdsAt` without materializing an - * element view. Plain elements are read in place. - */ -function everyNodeElementDeletion(node, offset, predicate) { - if (node.isSpan) { - const deletionIds = node.element.deletionIdsAt(offset); - for (let index = 0; index < deletionIds.length; index += 1) { - if (!predicate(deletionIds[index])) - return false; - } - return true; - } - const storage = node.element; - return everySequenceDeletion(Array.isArray(storage) ? storage[offset] : storage, predicate); -} -function elementNode(element) { - return element[SEQUENCE_NODE]; -} -function elementOffset(element) { - return element[SEQUENCE_OFFSET]; -} -function setElementLocation(element, node, offset) { - const located = element; - located[SEQUENCE_NODE] = node; - located[SEQUENCE_OFFSET] = offset; -} -function nodeLength(node) { - return node.ownCount; -} -function nodeLengthFromStorage(storage) { - return storage instanceof SequenceSpan - ? storage.length - : Array.isArray(storage) - ? storage.length - : 1; -} -function nodeElement(node, offset) { - if (node.isSpan) { - const element = node.element.elementAt(offset); - setElementLocation(element, node, offset); - return element; - } - const storage = node.element; - return Array.isArray(storage) ? storage[offset] : storage; -} -function nodePeer(node, offset) { - return node.isSpan - ? node.element.peerAt(offset) - : Array.isArray(node.element) - ? node.element[offset].id.peer - : node.element.id.peer; -} -function nodeCounter(node, offset) { - return node.isSpan - ? node.element.counterAt(offset) - : Array.isArray(node.element) - ? node.element[offset].id.counter - : node.element.id.counter; -} -function findNextIncludedPhysical(node, nodeStart, start, version) { - if (node === undefined || nodeStart + node.allCount <= start) - return undefined; - if (node.idRunCount === 1 && - (version.get(node.firstId.peer) ?? 0) <= node.firstId.counter) { - return undefined; - } - const leftCount = allCount(node.left); - const ownStart = nodeStart + leftCount; - const fromLeft = findNextIncludedPhysical(node.left, nodeStart, start, version); - if (fromLeft !== undefined) - return fromLeft; - const offsetStart = Math.max(0, start - ownStart); - if (offsetStart < nodeLength(node)) { - if (node.ownIdRunCount === 1) { - const peer = nodePeer(node, offsetStart); - const counter = nodeCounter(node, offsetStart); - if (counter < (version.get(peer) ?? 0)) { - return { - element: nodeElement(node, offsetStart), - index: ownStart + offsetStart, - }; - } - } - else { - for (let offset = offsetStart; offset < nodeLength(node); offset += 1) { - const peer = nodePeer(node, offset); - const counter = nodeCounter(node, offset); - if (counter < (version.get(peer) ?? 0)) { - return { element: nodeElement(node, offset), index: ownStart + offset }; - } - } - } - } - return findNextIncludedPhysical(node.right, ownStart + nodeLength(node), start, version); -} -function nodeLamport(node, offset) { - return node.isSpan - ? node.element.lamportAt(offset) - : Array.isArray(node.element) - ? node.element[offset].lamport - : node.element.lamport; -} -function nodeDeleted(node, offset) { - return node.isSpan - ? node.element.deletedAt(offset) - : Array.isArray(node.element) - ? node.element[offset].deleted - : node.element.deleted; -} -function setNodeDeleted(node, offset, deleted) { - if (node.isSpan) { - node.element.setDeletedAt(offset, deleted); - } - else if (Array.isArray(node.element)) { - node.element[offset].deleted = deleted; - } - else { - node.element.deleted = deleted; - } -} -function nodeMetrics(node, offset, metrics) { - return node.isSpan - ? node.element.metricsAt(offset) - : metrics(Array.isArray(node.element) - ? node.element[offset] - : node.element); -} -function sliceNodeStorage(storage, start, end) { - if (storage instanceof SequenceSpan) - return storage.slice(start, end); - const elements = Array.isArray(storage) ? storage.slice(start, end) : [storage]; - return elements.length === 1 ? elements[0] : elements; -} -function appendNodeElements(node, output) { - if (node.isSpan) { - const span = node.element; - for (let offset = 0; offset < span.length; offset += 1) { - output.push(nodeElement(node, offset)); - } - } - else if (Array.isArray(node.element)) - output.push(...node.element); - else - output.push(node.element); -} -function allCount(node) { - return node?.allCount ?? 0; -} -function visibleCount(node) { - return node?.visibleCount ?? 0; -} -function visibleMetric(node, metric) { - if (node === undefined) - return 0; - return metric === "utf16" ? node.visibleUtf16 : node.visibleUtf8; -} -function ownVisibleCount(node) { - return node.ownVisibleCount; -} -function visibleElementsBefore(node, offset) { - if (node.visibleOffsets !== undefined) - return node.visibleOffsets[offset]; - let count = 0; - for (let index = 0; index < offset; index += 1) { - if (!nodeDeleted(node, index)) - count += 1; - } - return count; -} -function ownVisibleMetric(node, metric, _metrics) { - return metric === "utf16" ? node.ownVisibleUtf16 : node.ownVisibleUtf8; -} -function visibleMetricBefore(node, offset, metric, metrics) { - const prefix = metric === "utf16" ? node.visibleUtf16Prefix : node.visibleUtf8Prefix; - if (prefix !== undefined) - return prefix[offset]; - let total = 0; - for (let index = 0; index < offset; index += 1) { - if (!nodeDeleted(node, index)) - total += nodeMetrics(node, index, metrics)[metric]; - } - return total; -} -function physicalOffsetAtVisibleIndex(node, index) { - if (node.ownVisibleCount === node.ownCount) - return index; - const prefix = node.visibleOffsets; - if (prefix === undefined) - return 0; - let low = 1; - let high = prefix.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (prefix[middle] > index) - high = middle; - else - low = middle + 1; - } - return low - 1; -} -function physicalOffsetAtMetricOffset(node, offset, metric) { - const prefix = metric === "utf16" ? node.visibleUtf16Prefix : node.visibleUtf8Prefix; - if (prefix === undefined) { - if (offset === 0) - return 0; - return offset === ownVisibleMetric(node, metric, listMetrics) ? 1 : undefined; - } - let low = 0; - let high = prefix.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (prefix[middle] >= offset) - high = middle; - else - low = middle + 1; - } - return prefix[low] === offset ? low : undefined; -} -function updateNodeElementVisibility(node, offset, element, deleted, metrics) { - const value = metrics(element); - const direction = deleted ? -1 : 1; - setNodeDeleted(node, offset, deleted); - node.ownVisibleCount += direction; - node.ownVisibleUtf16 += direction * value.utf16; - node.ownVisibleUtf8 += direction * value.utf8; - recomputeOwnIdRuns(node); - if (nodeLength(node) === 1) - return; - for (let index = offset + 1; index < node.visibleOffsets.length; index += 1) { - node.visibleOffsets[index] += direction; - node.visibleUtf16Prefix[index] += direction * value.utf16; - node.visibleUtf8Prefix[index] += direction * value.utf8; - } -} -function recomputeOwn(node, metrics) { - if (!node.isSpan) { - recomputeOwnElements(node, metrics); - return; - } - const length = nodeLength(node); - const firstPeer = nodePeer(node, 0); - const firstCounter = nodeCounter(node, 0); - node.ownFirstId = { peer: firstPeer, counter: firstCounter }; - node.ownLastId = - length === 1 - ? node.ownFirstId - : { - peer: nodePeer(node, length - 1), - counter: nodeCounter(node, length - 1), - }; - node.ownIdRunCount = 1; - let previousPeer = firstPeer; - let previousCounter = firstCounter; - for (let offset = 1; offset < length; offset += 1) { - const peer = nodePeer(node, offset); - const counter = nodeCounter(node, offset); - if (!sequenceIdScalarsContinue(previousPeer, previousCounter, peer, counter)) { - node.ownIdRunCount += 1; - } - previousPeer = peer; - previousCounter = counter; - } - if (length === 1) { - const value = nodeMetrics(node, 0, metrics); - node.ownUtf16 = value.utf16; - node.ownUtf8 = value.utf8; - node.visibleOffsets = undefined; - node.visibleUtf16Prefix = undefined; - node.visibleUtf8Prefix = undefined; - if (nodeDeleted(node, 0)) { - node.ownVisibleCount = 0; - node.ownVisibleUtf16 = 0; - node.ownVisibleUtf8 = 0; - node.ownFirstVisibleId = undefined; - node.ownLastVisibleId = undefined; - node.ownVisibleIdRunCount = 0; - } - else { - node.ownVisibleCount = 1; - node.ownVisibleUtf16 = value.utf16; - node.ownVisibleUtf8 = value.utf8; - node.ownFirstVisibleId = node.ownFirstId; - node.ownLastVisibleId = node.ownFirstId; - node.ownVisibleIdRunCount = 1; - } - return; - } - node.ownVisibleCount = 0; - node.ownVisibleUtf16 = 0; - node.ownVisibleUtf8 = 0; - node.ownUtf16 = 0; - node.ownUtf8 = 0; - node.ownFirstVisibleId = undefined; - node.ownLastVisibleId = undefined; - node.ownVisibleIdRunCount = 0; - const visibleOffsets = [0]; - const visibleUtf16Prefix = [0]; - const visibleUtf8Prefix = [0]; - let lastVisiblePeer; - let lastVisibleCounter = 0; - for (let offset = 0; offset < length; offset += 1) { - const peer = nodePeer(node, offset); - const counter = nodeCounter(node, offset); - const value = nodeMetrics(node, offset, metrics); - node.ownUtf16 += value.utf16; - node.ownUtf8 += value.utf8; - if (!nodeDeleted(node, offset)) { - if (node.ownVisibleCount === 0) { - node.ownFirstVisibleId = { peer, counter }; - node.ownVisibleIdRunCount = 1; - } - else if (!sequenceIdScalarsContinue(lastVisiblePeer, lastVisibleCounter, peer, counter)) { - node.ownVisibleIdRunCount += 1; - } - lastVisiblePeer = peer; - lastVisibleCounter = counter; - node.ownVisibleCount += 1; - node.ownVisibleUtf16 += value.utf16; - node.ownVisibleUtf8 += value.utf8; - } - visibleOffsets.push(node.ownVisibleCount); - visibleUtf16Prefix.push(node.ownVisibleUtf16); - visibleUtf8Prefix.push(node.ownVisibleUtf8); - } - node.ownLastVisibleId = - lastVisiblePeer === undefined - ? undefined - : { peer: lastVisiblePeer, counter: lastVisibleCounter }; - node.visibleOffsets = visibleOffsets; - node.visibleUtf16Prefix = visibleUtf16Prefix; - node.visibleUtf8Prefix = visibleUtf8Prefix; -} -function recomputeOwnElements(node, metrics) { - const storage = node.element; - const elements = Array.isArray(storage) ? storage : undefined; - const first = elements?.[0] ?? storage; - const last = elements?.at(-1) ?? storage; - node.ownFirstId = first.id; - node.ownLastId = last.id; - node.ownIdRunCount = 1; - if (elements !== undefined) { - for (let offset = 1; offset < elements.length; offset += 1) { - if (!sequenceIdsContinue(elements[offset - 1].id, elements[offset].id)) { - node.ownIdRunCount += 1; - } - } - } - if (elements === undefined) { - const value = metrics(storage); - node.ownUtf16 = value.utf16; - node.ownUtf8 = value.utf8; - node.visibleOffsets = undefined; - node.visibleUtf16Prefix = undefined; - node.visibleUtf8Prefix = undefined; - if (storage.deleted) { - node.ownVisibleCount = 0; - node.ownVisibleUtf16 = 0; - node.ownVisibleUtf8 = 0; - node.ownFirstVisibleId = undefined; - node.ownLastVisibleId = undefined; - node.ownVisibleIdRunCount = 0; - } - else { - node.ownVisibleCount = 1; - node.ownVisibleUtf16 = value.utf16; - node.ownVisibleUtf8 = value.utf8; - node.ownFirstVisibleId = storage.id; - node.ownLastVisibleId = storage.id; - node.ownVisibleIdRunCount = 1; - } - return; - } - node.ownVisibleCount = 0; - node.ownVisibleUtf16 = 0; - node.ownVisibleUtf8 = 0; - node.ownUtf16 = 0; - node.ownUtf8 = 0; - node.ownFirstVisibleId = undefined; - node.ownLastVisibleId = undefined; - node.ownVisibleIdRunCount = 0; - const visibleOffsets = [0]; - const visibleUtf16Prefix = [0]; - const visibleUtf8Prefix = [0]; - for (const element of elements) { - const value = metrics(element); - node.ownUtf16 += value.utf16; - node.ownUtf8 += value.utf8; - if (!element.deleted) { - if (node.ownVisibleCount === 0) { - node.ownFirstVisibleId = element.id; - node.ownVisibleIdRunCount = 1; - } - else if (!sequenceIdsContinue(node.ownLastVisibleId, element.id)) { - node.ownVisibleIdRunCount += 1; - } - node.ownLastVisibleId = element.id; - node.ownVisibleCount += 1; - node.ownVisibleUtf16 += value.utf16; - node.ownVisibleUtf8 += value.utf8; - } - visibleOffsets.push(node.ownVisibleCount); - visibleUtf16Prefix.push(node.ownVisibleUtf16); - visibleUtf8Prefix.push(node.ownVisibleUtf8); - } - node.visibleOffsets = visibleOffsets; - node.visibleUtf16Prefix = visibleUtf16Prefix; - node.visibleUtf8Prefix = visibleUtf8Prefix; -} -function recomputeVisibility(node) { - const left = node.left; - const right = node.right; - node.visibleCount = - (left?.visibleCount ?? 0) + node.ownVisibleCount + (right?.visibleCount ?? 0); - node.visibleUtf16 = - (left?.visibleUtf16 ?? 0) + node.ownVisibleUtf16 + (right?.visibleUtf16 ?? 0); - node.visibleUtf8 = - (left?.visibleUtf8 ?? 0) + node.ownVisibleUtf8 + (right?.visibleUtf8 ?? 0); - node.firstVisibleId = - left?.firstVisibleId ?? node.ownFirstVisibleId ?? right?.firstVisibleId; - node.lastVisibleId = - right?.lastVisibleId ?? node.ownLastVisibleId ?? left?.lastVisibleId; - node.visibleIdRunCount = - (left?.visibleIdRunCount ?? 0) + - node.ownVisibleIdRunCount + - (right?.visibleIdRunCount ?? 0); - const leftLastVisible = left?.lastVisibleId; - const ownFirstVisible = node.ownFirstVisibleId; - if (leftLastVisible !== undefined && - ownFirstVisible !== undefined && - leftLastVisible.peer === ownFirstVisible.peer && - leftLastVisible.counter + 1 === ownFirstVisible.counter) { - node.visibleIdRunCount -= 1; - } - const beforeRight = node.ownLastVisibleId ?? leftLastVisible; - const rightFirstVisible = right?.firstVisibleId; - if (beforeRight !== undefined && - rightFirstVisible !== undefined && - beforeRight.peer === rightFirstVisible.peer && - beforeRight.counter + 1 === rightFirstVisible.counter) { - node.visibleIdRunCount -= 1; - } -} -function recompute(node, _metrics) { - const left = node.left; - const right = node.right; - if (left === undefined && right === undefined) { - node.allCount = nodeLength(node); - node.visibleCount = node.ownVisibleCount; - node.visibleUtf16 = node.ownVisibleUtf16; - node.visibleUtf8 = node.ownVisibleUtf8; - node.firstVisibleId = node.ownFirstVisibleId; - node.lastVisibleId = node.ownLastVisibleId; - node.visibleIdRunCount = node.ownVisibleIdRunCount; - node.firstId = node.ownFirstId; - node.lastId = node.ownLastId; - node.idRunCount = node.ownIdRunCount; - node.allUtf16 = node.ownUtf16; - node.allUtf8 = node.ownUtf8; - return; - } - node.allCount = (left?.allCount ?? 0) + nodeLength(node) + (right?.allCount ?? 0); - node.allUtf16 = (left?.allUtf16 ?? 0) + node.ownUtf16 + (right?.allUtf16 ?? 0); - node.allUtf8 = (left?.allUtf8 ?? 0) + node.ownUtf8 + (right?.allUtf8 ?? 0); - recomputeVisibility(node); - node.firstId = left?.firstId ?? node.ownFirstId; - node.lastId = right?.lastId ?? node.ownLastId; - node.idRunCount = - (left?.idRunCount ?? 0) + node.ownIdRunCount + (right?.idRunCount ?? 0); - if (left !== undefined && - left.lastId.peer === node.ownFirstId.peer && - left.lastId.counter + 1 === node.ownFirstId.counter) { - node.idRunCount -= 1; - } - if (right !== undefined && - node.ownLastId.peer === right.firstId.peer && - node.ownLastId.counter + 1 === right.firstId.counter) { - node.idRunCount -= 1; - } - if (left !== undefined) - left.parent = node; - if (right !== undefined) - right.parent = node; -} -function applyNodeDeleted(node, metrics) { - if (node.lazyDeleted && node.visibleCount === 0) - return; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - setNodeDeleted(node, offset, true); - } - recomputeOwn(node, metrics); - node.visibleCount = 0; - node.visibleUtf16 = 0; - node.visibleUtf8 = 0; - node.firstVisibleId = undefined; - node.lastVisibleId = undefined; - node.visibleIdRunCount = 0; - node.lazyDeleted = true; - node.lazyVisible = false; -} -function applyNodeVisible(node, metrics) { - if (node.lazyVisible && node.visibleCount === node.allCount) - return; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - setNodeDeleted(node, offset, false); - } - recomputeOwn(node, metrics); - node.visibleCount = node.allCount; - node.visibleUtf16 = node.allUtf16; - node.visibleUtf8 = node.allUtf8; - node.firstVisibleId = node.firstId; - node.lastVisibleId = node.lastId; - node.visibleIdRunCount = node.idRunCount; - node.lazyDeleted = false; - node.lazyVisible = true; -} -function pushNodeDeletion(node, metrics) { - if (!node.lazyDeleted && !node.lazyVisible) - return; - if (node.left !== undefined) { - if (node.lazyDeleted) - applyNodeDeleted(node.left, metrics); - else - applyNodeVisible(node.left, metrics); - } - if (node.right !== undefined) { - if (node.lazyDeleted) - applyNodeDeleted(node.right, metrics); - else - applyNodeVisible(node.right, metrics); - } - node.lazyDeleted = false; - node.lazyVisible = false; -} -function materializeElementDeletion(element, metrics) { - const node = elementNode(element); - if (node === undefined) - return; - materializeNodeDeletion(node, metrics); -} -function materializeNodeDeletion(node, metrics) { - const path = []; - let current = node; - while (current !== undefined) { - path.push(current); - current = current.parent; - } - for (let index = path.length - 1; index >= 0; index -= 1) { - pushNodeDeletion(path[index], metrics); - } -} -function materializeAllDeletions(node, metrics) { - if (node === undefined) - return; - pushNodeDeletion(node, metrics); - materializeAllDeletions(node.left, metrics); - materializeAllDeletions(node.right, metrics); -} -function recomputeOwnIdRuns(node) { - node.ownFirstVisibleId = undefined; - node.ownLastVisibleId = undefined; - node.ownVisibleIdRunCount = 0; - let lastPeer; - let lastCounter = 0; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (nodeDeleted(node, offset)) - continue; - const peer = nodePeer(node, offset); - const counter = nodeCounter(node, offset); - if (lastPeer === undefined) { - node.ownFirstVisibleId = { peer, counter }; - node.ownVisibleIdRunCount = 1; - } - else if (!sequenceIdScalarsContinue(lastPeer, lastCounter, peer, counter)) { - node.ownVisibleIdRunCount += 1; - } - lastPeer = peer; - lastCounter = counter; - } - node.ownLastVisibleId = - lastPeer === undefined ? undefined : { peer: lastPeer, counter: lastCounter }; -} -function sequenceIdsContinue(left, right) { - return left.peer === right.peer && left.counter + 1 === right.counter; -} -function sequenceIdScalarsContinue(leftPeer, leftCounter, rightPeer, rightCounter) { - return leftPeer === rightPeer && leftCounter + 1 === rightCounter; -} -function normalizeSequenceIdRuns(runs) { - const sorted = [...runs].sort((left, right) => left.start.peer < right.start.peer - ? -1 - : left.start.peer > right.start.peer - ? 1 - : left.start.counter - right.start.counter); - const output = []; - for (const run of sorted) { - const previous = output.at(-1); - if (previous !== undefined && - previous.start.peer === run.start.peer && - run.start.counter <= previous.start.counter + previous.length) { - previous.length = - Math.max(previous.start.counter + previous.length, run.start.counter + run.length) - previous.start.counter; - } - else { - output.push({ start: { ...run.start }, length: run.length }); - } - } - return output; -} -function recomputeToRoot(node, metrics, includePhysical = true) { - let current = node; - while (current !== undefined) { - if (includePhysical) - recompute(current, metrics); - else - recomputeVisibility(current); - current = current.parent; - } -} -function recomputeAffected(node, affected, metrics) { - if (node === undefined || !affected.has(node)) - return; - recomputeAffected(node.left, affected, metrics); - recomputeAffected(node.right, affected, metrics); - recompute(node, metrics); -} -function merge(left, right, metrics) { - if (left === undefined) { - if (right !== undefined) - right.parent = undefined; - return right; - } - if (right === undefined) { - left.parent = undefined; - return left; - } - if (left.priority <= right.priority) { - pushNodeDeletion(left, metrics); - left.right = merge(left.right, right, metrics); - recompute(left, metrics); - left.parent = undefined; - return left; - } - pushNodeDeletion(right, metrics); - right.left = merge(left, right.left, metrics); - recompute(right, metrics); - right.parent = undefined; - return right; -} -function visitInOrder(root, visit) { - const stack = []; - let node = root; - while (node !== undefined || stack.length > 0) { - while (node !== undefined) { - stack.push(node); - node = node.left; - } - node = stack.pop(); - visit(node); - node = node.right; - } -} -function visitVisibleIdRuns(node, start, end, metrics, visit) { - if (node === undefined || start >= end) - return; - const total = visibleCount(node); - if (start === 0 && - end === total && - node.visibleIdRunCount === 1 && - node.firstVisibleId !== undefined) { - visit(node.firstVisibleId, total); - return; - } - pushNodeDeletion(node, metrics); - const leftCount = visibleCount(node.left); - if (start < leftCount) { - visitVisibleIdRuns(node.left, start, Math.min(end, leftCount), metrics, visit); - } - const ownCount = ownVisibleCount(node); - const ownStart = Math.max(0, start - leftCount); - const ownEnd = Math.min(ownCount, end - leftCount); - if (ownStart < ownEnd) { - if (ownStart === 0 && - ownEnd === ownCount && - node.ownVisibleIdRunCount === 1 && - node.ownFirstVisibleId !== undefined) { - visit(node.ownFirstVisibleId, ownCount); - } - else { - let visibleOffset = 0; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (nodeDeleted(node, offset)) - continue; - if (visibleOffset >= ownStart && visibleOffset < ownEnd) { - // The visit callback may retain the id, so each run start is a - // fresh object; only the element view allocation is avoided. - visit({ peer: nodePeer(node, offset), counter: nodeCounter(node, offset) }, 1); - } - visibleOffset += 1; - if (visibleOffset >= ownEnd) - break; - } - } - } - const rightStart = leftCount + ownCount; - if (end > rightStart) { - visitVisibleIdRuns(node.right, Math.max(0, start - rightStart), end - rightStart, metrics, visit); - } -} -function counterRangesByPeer(runs) { - const byPeer = new Map(); - for (const run of runs) { - if (run.length <= 0) - continue; - let ranges = byPeer.get(run.start.peer); - if (ranges === undefined) { - ranges = []; - byPeer.set(run.start.peer, ranges); - } - ranges.push({ start: run.start.counter, end: run.start.counter + run.length }); - } - for (const ranges of byPeer.values()) { - ranges.sort((left, right) => left.start - right.start); - let write = 0; - for (const range of ranges) { - const previous = ranges[write - 1]; - if (previous !== undefined && range.start <= previous.end) { - previous.end = Math.max(previous.end, range.end); - } - else { - ranges[write++] = range; - } - } - ranges.length = write; - } - return byPeer; -} -function counterRangeRelation(targets, peer, start, end) { - const ranges = targets.get(peer); - if (ranges === undefined) - return 0; - let low = 0; - let high = ranges.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (ranges[middle].end <= start) - low = middle + 1; - else - high = middle; - } - const range = ranges[low]; - if (range === undefined || range.start >= end) - return 0; - return range.start <= start && range.end >= end ? 1 : -1; -} -function mapCounterRangeRelation(targets) { - return (peer, start, end) => counterRangeRelation(targets, peer, start, end); -} -/** - * Scalar relation for the common single-run delete: avoids building a - * Map and the per-node bigint-keyed lookup. - */ -function singleCounterRangeRelation(targetPeer, targetStart, targetEnd) { - return (peer, start, end) => { - if (peer !== targetPeer || targetStart >= end || targetEnd <= start) - return 0; - return targetStart <= start && targetEnd >= end ? 1 : -1; - }; -} -function visitVisibleMetricRangesForIds(node, base, metric, targets, metrics, visit) { - if (node === undefined || node.visibleCount === 0) - return; - if (node.visibleIdRunCount === 1 && node.firstVisibleId !== undefined) { - const relation = counterRangeRelation(targets, node.firstVisibleId.peer, node.firstVisibleId.counter, node.firstVisibleId.counter + node.visibleCount); - if (relation === 0) - return; - if (relation === 1) { - visit(base, base + visibleMetric(node, metric)); - return; - } - } - pushNodeDeletion(node, metrics); - visitVisibleMetricRangesForIds(node.left, base, metric, targets, metrics, visit); - let ownBase = base + visibleMetric(node.left, metric); - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (nodeDeleted(node, offset)) - continue; - const length = nodeMetrics(node, offset, metrics)[metric]; - const counter = nodeCounter(node, offset); - if (counterRangeRelation(targets, nodePeer(node, offset), counter, counter + 1) === 1) { - visit(ownBase, ownBase + length); - } - ownBase += length; - } - visitVisibleMetricRangesForIds(node.right, ownBase, metric, targets, metrics, visit); -} -function countPhysicalIdsInRanges(node, targets) { - if (node === undefined) - return 0; - if (node.idRunCount === 1) { - const relation = counterRangeRelation(targets, node.firstId.peer, node.firstId.counter, node.firstId.counter + node.allCount); - if (relation === 0) - return 0; - if (relation === 1) - return node.allCount; - } - let count = countPhysicalIdsInRanges(node.left, targets); - for (let offset = 0; offset < nodeLength(node); offset += 1) { - const counter = nodeCounter(node, offset); - if (counterRangeRelation(targets, nodePeer(node, offset), counter, counter + 1) === 1) { - count += 1; - } - } - return count + countPhysicalIdsInRanges(node.right, targets); -} -function markIdRunsDeleted(node, relate, metrics) { - if (node === undefined) - return { found: false, changed: false }; - if (node.idRunCount === 1) { - const relation = relate(node.firstId.peer, node.firstId.counter, node.firstId.counter + node.allCount); - if (relation === 0) - return { found: false, changed: false }; - if (relation === 1) { - const changed = node.visibleCount > 0; - applyNodeDeleted(node, metrics); - return { found: true, changed }; - } - } - pushNodeDeletion(node, metrics); - const left = markIdRunsDeleted(node.left, relate, metrics); - let ownFound = false; - let ownChanged = false; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - const counter = nodeCounter(node, offset); - if (relate(nodePeer(node, offset), counter, counter + 1) !== 1) { - continue; - } - ownFound = true; - if (!nodeDeleted(node, offset)) { - setNodeDeleted(node, offset, true); - ownChanged = true; - } - } - if (ownChanged) - recomputeOwn(node, metrics); - const right = markIdRunsDeleted(node.right, relate, metrics); - const changed = left.changed || ownChanged || right.changed; - if (changed) - recomputeVisibility(node); - return { - found: left.found || ownFound || right.found, - changed, - }; -} -function markIdRunsVisible(node, relate, metrics) { - if (node === undefined) - return { found: false, changed: false }; - if (node.idRunCount === 1) { - const relation = relate(node.firstId.peer, node.firstId.counter, node.firstId.counter + node.allCount); - if (relation === 0) - return { found: false, changed: false }; - if (relation === 1) { - const changed = node.visibleCount < node.allCount; - applyNodeVisible(node, metrics); - return { found: true, changed }; - } - } - pushNodeDeletion(node, metrics); - const left = markIdRunsVisible(node.left, relate, metrics); - let ownFound = false; - let ownChanged = false; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - const counter = nodeCounter(node, offset); - if (relate(nodePeer(node, offset), counter, counter + 1) !== 1) { - continue; - } - ownFound = true; - if (nodeDeleted(node, offset)) { - setNodeDeleted(node, offset, false); - ownChanged = true; - } - } - if (ownChanged) - recomputeOwn(node, metrics); - const right = markIdRunsVisible(node.right, relate, metrics); - const changed = left.changed || ownChanged || right.changed; - if (changed) - recomputeVisibility(node); - return { - found: left.found || ownFound || right.found, - changed, - }; -} -function visitVisibleRange(node, start, end, metrics, visit) { - if (node === undefined || start >= end) - return true; - pushNodeDeletion(node, metrics); - const leftCount = visibleCount(node.left); - if (start < leftCount) { - if (!visitVisibleRange(node.left, start, Math.min(end, leftCount), metrics, visit)) { - return false; - } - } - let ownOffset = leftCount; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (nodeDeleted(node, offset)) - continue; - if (start <= ownOffset && - end > ownOffset && - visit(nodeElement(node, offset)) === false) { - return false; - } - ownOffset += 1; - if (ownOffset >= end) - break; - } - if (end > ownOffset) { - return visitVisibleRange(node.right, Math.max(0, start - ownOffset), end - ownOffset, metrics, visit); - } - return true; -} -function collectCausalRange(node, start, end, output, count, isVisible, metrics) { - if (node === undefined || start >= end) - return; - pushNodeDeletion(node, metrics); - const leftCount = count(node.left); - if (start < leftCount) { - collectCausalRange(node.left, start, Math.min(end, leftCount), output, count, isVisible, metrics); - } - let ownOffset = leftCount; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (!isVisible(node, offset)) - continue; - if (start <= ownOffset && end > ownOffset) - output.push(nodeElement(node, offset)); - ownOffset += 1; - if (ownOffset >= end) - break; - } - if (end > ownOffset) { - collectCausalRange(node.right, Math.max(0, start - ownOffset), end - ownOffset, output, count, isVisible, metrics); - } -} -function collectCausalIdRuns(node, start, end, output, count, isVisible) { - if (node === undefined || start >= end) - return; - const nodeCount = count(node); - if (start === 0 && - end === nodeCount && - nodeCount === node.allCount && - node.idRunCount === 1) { - appendSequenceIdRun(output, node.firstId, node.allCount); - return; - } - const leftCount = count(node.left); - if (start < leftCount) { - collectCausalIdRuns(node.left, start, Math.min(end, leftCount), output, count, isVisible); - } - let ownOffset = leftCount; - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (!isVisible(node, offset)) - continue; - if (start <= ownOffset && end > ownOffset) { - // appendSequenceIdRun copies the start id before retaining it. - scratchSequenceRunId.peer = nodePeer(node, offset); - scratchSequenceRunId.counter = nodeCounter(node, offset); - appendSequenceIdRun(output, scratchSequenceRunId, 1); - } - ownOffset += 1; - if (ownOffset >= end) - break; - } - if (end > ownOffset) { - collectCausalIdRuns(node.right, Math.max(0, start - ownOffset), end - ownOffset, output, count, isVisible); - } -} -function appendSequenceIdRun(output, start, length) { - if (length <= 0) - return; - const previous = output.at(-1); - if (previous !== undefined && - previous.start.peer === start.peer && - previous.start.counter + previous.length === start.counter) { - output[output.length - 1] = { - start: previous.start, - length: previous.length + length, - }; - } - else { - output.push({ start: { ...start }, length }); - } -} -function firstPhysicalElement(node) { - if (node === undefined) - return undefined; - while (node.left !== undefined) - node = node.left; - return nodeElement(node, 0); -} -function writeFirstPhysicalId(node, output) { - if (node === undefined) { - output.rightPeer = undefined; - output.rightCounter = 0; - return; - } - while (node.left !== undefined) - node = node.left; - output.rightPeer = nodePeer(node, 0); - output.rightCounter = nodeCounter(node, 0); -} -function nextPhysicalElement(node, offset) { - if (offset + 1 < nodeLength(node)) - return nodeElement(node, offset + 1); - const fromRight = firstPhysicalElement(node.right); - if (fromRight !== undefined) - return fromRight; - let current = node; - while (current.parent !== undefined) { - if (current === current.parent.left) - return nodeElement(current.parent, 0); - current = current.parent; - } - return undefined; -} -function writeNextPhysicalId(node, offset, output) { - if (offset + 1 < nodeLength(node)) { - output.rightPeer = nodePeer(node, offset + 1); - output.rightCounter = nodeCounter(node, offset + 1); - return; - } - if (node.right !== undefined) { - writeFirstPhysicalId(node.right, output); - return; - } - let current = node; - while (current.parent !== undefined) { - if (current === current.parent.left) { - output.rightPeer = nodePeer(current.parent, 0); - output.rightCounter = nodeCounter(current.parent, 0); - return; - } - current = current.parent; - } - output.rightPeer = undefined; - output.rightCounter = 0; -} -function firstVisibleElement(node, metrics) { - while (node !== undefined && node.visibleCount > 0) { - pushNodeDeletion(node, metrics); - if (visibleCount(node.left) > 0) { - node = node.left; - continue; - } - for (let offset = 0; offset < nodeLength(node); offset += 1) { - if (!nodeDeleted(node, offset)) - return nodeElement(node, offset); - } - node = node.right; - } - return undefined; -} -function lastVisibleElement(node, metrics) { - while (node !== undefined && node.visibleCount > 0) { - pushNodeDeletion(node, metrics); - if (visibleCount(node.right) > 0) { - node = node.right; - continue; - } - for (let offset = nodeLength(node) - 1; offset >= 0; offset -= 1) { - if (!nodeDeleted(node, offset)) - return nodeElement(node, offset); - } - node = node.left; - } - return undefined; -} -function rightmost(node) { - while (node.right !== undefined) - node = node.right; - return node; -} -function versionsEqual(left, right) { - if (left === undefined || left.size !== right.size) - return false; - for (const [peer, counter] of left) { - if (right.get(peer) !== counter) - return false; - } - return true; -} -const MAX_COUNTER_DENSE_GAP = 4096; -const COUNTER_PAGE_SIZE = 1024; -class PagedCounterStore { - #pages = new Map(); - #pageIndexes = []; - set(counter, value) { - // counter is non-negative; COUNTER_PAGE_SIZE is 1,024. - const pageIndex = (counter / COUNTER_PAGE_SIZE) | 0; - let page = this.#pages.get(pageIndex); - if (page === undefined) { - page = []; - this.#pages.set(pageIndex, page); - const index = lowerBoundNumber(this.#pageIndexes, pageIndex); - this.#pageIndexes.splice(index, 0, pageIndex); - } - page[counter - pageIndex * COUNTER_PAGE_SIZE] = value; - } - some(start, end, predicate) { - if (start >= end) - return false; - const firstPage = (start / COUNTER_PAGE_SIZE) | 0; - const lastPage = ((end - 1) / COUNTER_PAGE_SIZE) | 0; - for (let index = lowerBoundNumber(this.#pageIndexes, firstPage); index < this.#pageIndexes.length && this.#pageIndexes[index] <= lastPage; index += 1) { - const pageIndex = this.#pageIndexes[index]; - const page = this.#pages.get(pageIndex); - const pageStart = pageIndex * COUNTER_PAGE_SIZE; - const offsetStart = Math.max(0, start - pageStart); - const offsetEnd = Math.min(page.length, end - pageStart); - for (let offset = offsetStart; offset < offsetEnd; offset += 1) { - const value = page[offset]; - if (value !== undefined && predicate(value)) - return true; - } - } - return false; - } -} -class CounterStore { - #dense = []; - #sparse = new Map(); - #sparseCounters = new CounterIndex(); - get(counter) { - return counter < this.#dense.length - ? this.#dense[counter] - : this.#sparse.get(counter); - } - has(counter) { - return this.get(counter) !== undefined; - } - set(counter, value) { - if (counter < 4294967295 && counter <= this.#dense.length + MAX_COUNTER_DENSE_GAP) { - this.#dense[counter] = value; - return; - } - if (!this.#sparse.has(counter)) - this.#sparseCounters.add(counter); - this.#sparse.set(counter, value); - } - reserveDense(endExclusive) { - if (this.#sparse.size > 0) { - throw new Error("cannot reserve dense sequence counters after sparse insertion"); - } - if (endExclusive > this.#dense.length) - this.#dense.length = endExclusive; - } - forEach(start, end, visit) { - if (start >= end) - return; - const denseEnd = Math.min(end, this.#dense.length); - for (let counter = Math.max(0, start); counter < denseEnd; counter += 1) { - const value = this.#dense[counter]; - if (value !== undefined) - visit(value, counter); - } - this.#sparseCounters.forEach(start, end, (counter) => { - visit(this.#sparse.get(counter), counter); - }); - } -} -class CounterIndex { - #ranges = []; - #singletons = []; - #outOfOrder = new CounterRangeTree(); - #lastCounter; - #tailIsRange = false; - add(counter) { - const last = this.#lastCounter; - if (last === undefined) { - this.#singletons.push(counter); - this.#lastCounter = counter; - return; - } - if (counter > last) { - if (counter === last + 1) { - if (this.#tailIsRange) { - this.#ranges[this.#ranges.length - 1] = counter + 1; - } - else { - this.#ranges.push(this.#singletons.pop(), counter + 1); - this.#tailIsRange = true; - } - } - else { - this.#singletons.push(counter); - this.#tailIsRange = false; - } - this.#lastCounter = counter; - return; - } - if (counter === last || containsCounter(this.#ranges, this.#singletons, counter)) { - return; - } - this.#outOfOrder.add(counter); - } - forEach(start, end, visit) { - if (start >= end) - return; - const index = lowerBoundCounterRange(this.#ranges, start); - if (index > 0) { - const previousEnd = this.#ranges[index * 2 - 1]; - if (previousEnd > start) { - visitCounterInterval(this.#ranges[(index - 1) * 2], previousEnd, start, end, visit); - } - } - for (let rangeIndex = index; rangeIndex < this.#ranges.length / 2 && this.#ranges[rangeIndex * 2] < end; rangeIndex += 1) { - visitCounterInterval(this.#ranges[rangeIndex * 2], this.#ranges[rangeIndex * 2 + 1], start, end, visit); - } - for (let singletonIndex = lowerBoundNumber(this.#singletons, start); singletonIndex < this.#singletons.length && this.#singletons[singletonIndex] < end; singletonIndex += 1) { - visit(this.#singletons[singletonIndex]); - } - this.#outOfOrder.forEach(start, end, visit); - } -} -class CounterRangeTree { - #ranges = new OrderedIndex((left, right) => left.start - right.start); - add(counter) { - const range = { start: counter, end: counter + 1 }; - const index = this.#ranges.lowerBound(range); - const previous = this.#ranges.at(index - 1); - if (previous !== undefined && counter < previous.end) - return; - const next = this.#ranges.at(index); - if (next?.start === counter) - return; - const extendsPrevious = previous?.end === counter; - const extendsNext = next?.start === counter + 1; - if (extendsPrevious && extendsNext) { - previous.end = next.end; - this.#ranges.delete(next); - } - else if (extendsPrevious) { - previous.end += 1; - } - else if (extendsNext) { - this.#ranges.delete(next); - next.start = counter; - this.#ranges.add(next); - } - else { - this.#ranges.add(range); - } - } - forEach(start, end, visit) { - if (start >= end) - return; - const lower = { start, end: start + 1 }; - const index = this.#ranges.lowerBound(lower); - const previous = this.#ranges.at(index - 1); - if (previous !== undefined && previous.end > start) { - visitCounterRange(previous, start, end, visit); - } - this.#ranges.forEachFrom(lower, (range) => { - if (range.start >= end) - return false; - visitCounterRange(range, start, end, visit); - }); - } -} -function lowerBoundCounterRange(ranges, counter) { - let low = 0; - let high = ranges.length / 2; - while (low < high) { - const middle = (low + high) >>> 1; - if (ranges[middle * 2] < counter) - low = middle + 1; - else - high = middle; - } - return low; -} -function lowerBoundNumber(values, value) { - let low = 0; - let high = values.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (values[middle] < value) - low = middle + 1; - else - high = middle; - } - return low; -} -function containsCounter(ranges, singletons, counter) { - const singletonIndex = lowerBoundNumber(singletons, counter); - if (singletonIndex < singletons.length && singletons[singletonIndex] === counter) { - return true; - } - const rangeIndex = lowerBoundCounterRange(ranges, counter); - if (rangeIndex * 2 < ranges.length && ranges[rangeIndex * 2] === counter) - return true; - return rangeIndex > 0 && counter < ranges[rangeIndex * 2 - 1]; -} -function visitCounterInterval(rangeStart, rangeEnd, start, end, visit) { - const boundedEnd = Math.min(rangeEnd, end); - for (let counter = Math.max(rangeStart, start); counter < boundedEnd; counter += 1) { - visit(counter); - } -} -function visitCounterRange(range, start, end, visit) { - const rangeEnd = Math.min(range.end, end); - for (let counter = Math.max(range.start, start); counter < rangeEnd; counter += 1) { - visit(counter); - } -} diff --git a/loro-js/src/runtime/text-style-index.js b/loro-js/src/runtime/text-style-index.js deleted file mode 100644 index 731d597a1..000000000 --- a/loro-js/src/runtime/text-style-index.js +++ /dev/null @@ -1,336 +0,0 @@ -import { OrderedIndex } from "./ordered-index"; -export class TextStyleIndex { - #segmentsByPeer = new Map(); - #emptyMetas = new Map(); - get isEmpty() { - return this.#segmentsByPeer.size === 0; - } - add(runs, key, meta) { - for (const run of normalizeRuns(runs)) { - const end = run.start.counter + run.length; - if (run.length <= 0) - continue; - const segments = this.#segments(run.start.peer); - this.#ensureBoundary(segments, run.start.counter); - this.#ensureBoundary(segments, end); - const existing = []; - const first = segments._lowerBoundBy((segment) => segment.start - run.start.counter); - segments._forEachFromIndex(first, (segment) => { - if (segment.start >= end) - return false; - existing.push(segment); - }); - let cursor = run.start.counter; - for (const segment of existing) { - if (cursor < segment.start) { - const gap = { - start: cursor, - end: segment.start, - histories: new Map(), - latestMetas: undefined, - }; - this.#insertMeta(gap, key, meta); - segments.add(gap); - } - this.#insertMeta(segment, key, meta); - cursor = segment.end; - } - if (cursor < end) { - const gap = { - start: cursor, - end, - histories: new Map(), - latestMetas: undefined, - }; - this.#insertMeta(gap, key, meta); - segments.add(gap); - } - } - } - historyAt(id, key) { - return this.#segmentAt(id)?.histories.get(key); - } - metasAt(id, version) { - const segment = this.#segmentAt(id); - if (segment === undefined) - return this.#emptyMetas; - if (version === undefined) - return this.#latestMetas(segment); - const metas = new Map(); - for (const [key, history] of segment.histories) { - const meta = latestIncluded(history, version); - if (meta !== undefined) - metas.set(key, meta); - } - return metas; - } - resolver(version) { - const lastByPeer = new Map(); - const resolved = new Map(); - return (id) => { - let lookup = lastByPeer.get(id.peer); - if (lookup === undefined || id.counter < lookup.start || id.counter >= lookup.end) { - lookup = this.#lookupAt(id); - lastByPeer.set(id.peer, lookup); - } - const segment = lookup.segment; - if (segment === undefined) - return this.#emptyMetas; - if (version === undefined) - return this.#latestMetas(segment); - const cached = resolved.get(segment); - if (cached !== undefined) - return cached; - const metas = new Map(); - for (const [key, history] of segment.histories) { - const meta = latestIncluded(history, version); - if (meta !== undefined) - metas.set(key, meta); - } - resolved.set(segment, metas); - return metas; - }; - } - rangeHasKey(runs, key, version) { - for (const run of normalizeRuns(runs)) { - const segments = this.#segmentsByPeer.get(run.start.peer); - if (segments === undefined) - continue; - const end = run.start.counter + run.length; - const first = Math.max(0, segments._lowerBoundBy((segment) => segment.start - run.start.counter) - 1); - let found = false; - segments._forEachFromIndex(first, (segment) => { - if (segment.start >= end) - return false; - if (segment.end <= run.start.counter) - return; - const meta = latestIncluded(segment.histories.get(key), version); - if (meta !== undefined && meta.value !== null) { - found = true; - return false; - } - }); - if (found) - return true; - } - return false; - } - transitions(runs, key, beforeVersion, afterVersion) { - const transitions = []; - const append = (start, length, before, after) => { - if (before === after || length === 0) - return; - const previous = transitions.at(-1); - if (previous !== undefined && - previous.before === before && - previous.after === after && - previous.run.start.peer === start.peer && - previous.run.start.counter + previous.run.length === start.counter) { - transitions[transitions.length - 1] = { - run: { start: previous.run.start, length: previous.run.length + length }, - before, - after, - }; - } - else { - transitions.push({ run: { start: { ...start }, length }, before, after }); - } - }; - for (const run of normalizeRuns(runs)) { - const segments = this.#segmentsByPeer.get(run.start.peer); - if (segments === undefined) - continue; - const end = run.start.counter + run.length; - const first = Math.max(0, segments._lowerBoundBy((segment) => segment.start - run.start.counter) - 1); - segments._forEachFromIndex(first, (segment) => { - if (segment.start >= end) - return false; - const start = Math.max(run.start.counter, segment.start); - const segmentEnd = Math.min(end, segment.end); - if (start >= segmentEnd) - return; - append({ peer: run.start.peer, counter: start }, segmentEnd - start, latestIncluded(segment.histories.get(key), beforeVersion), latestIncluded(segment.histories.get(key), afterVersion)); - }); - } - return transitions; - } - runsContainMeta(runs, key, id) { - for (const run of runs) { - const segments = this.#segmentsByPeer.get(run.start.peer); - if (segments === undefined) - return false; - const end = run.start.counter + run.length; - let cursor = run.start.counter; - const first = Math.max(0, segments._lowerBoundBy((segment) => segment.start - cursor) - 1); - let covered = true; - segments._forEachFromIndex(first, (segment) => { - if (cursor >= end) - return false; - if (segment.start > cursor || segment.start >= end) { - covered = false; - return false; - } - if (segment.end <= cursor) - return; - if (!segment.histories - .get(key) - ?.some((item) => item.startId.peer === id.peer && item.startId.counter === id.counter)) { - covered = false; - return false; - } - cursor = Math.min(end, segment.end); - }); - if (!covered || cursor < end) - return false; - } - return true; - } - reset() { - this.#segmentsByPeer.clear(); - } - #segments(peer) { - let segments = this.#segmentsByPeer.get(peer); - if (segments === undefined) { - segments = new OrderedIndex((left, right) => left.start - right.start); - this.#segmentsByPeer.set(peer, segments); - } - return segments; - } - #segmentAt(id) { - return this.#lookupAt(id).segment; - } - #lookupAt(id) { - const segments = this.#segmentsByPeer.get(id.peer); - if (segments === undefined) { - return { - start: Number.MIN_SAFE_INTEGER, - end: Number.MAX_SAFE_INTEGER, - segment: undefined, - }; - } - const nextIndex = segments._lowerBoundBy((segment) => segment.start - (id.counter + 1)); - const previous = segments.at(nextIndex - 1); - if (previous !== undefined && id.counter < previous.end) { - return { start: previous.start, end: previous.end, segment: previous }; - } - return { - start: previous?.end ?? Number.MIN_SAFE_INTEGER, - end: segments.at(nextIndex)?.start ?? Number.MAX_SAFE_INTEGER, - segment: undefined, - }; - } - #ensureBoundary(segments, counter) { - const index = segments._lowerBoundBy((segment) => segment.start - counter); - if (segments.at(index)?.start === counter) - return; - const previous = segments.at(index - 1); - if (previous === undefined || counter >= previous.end) - return; - segments.delete(previous); - const right = { - start: counter, - end: previous.end, - histories: cloneHistories(previous.histories), - latestMetas: previous.latestMetas, - }; - previous.end = counter; - segments.add(previous); - segments.add(right); - } - #insertMeta(segment, key, meta) { - let history = segment.histories.get(key); - if (history === undefined) { - history = []; - segment.histories.set(key, history); - } - const index = lowerBoundMeta(history, meta); - if (sameMeta(history[index], meta)) - return; - history.splice(index, 0, meta); - segment.latestMetas = undefined; - } - #latestMetas(segment) { - if (segment.latestMetas !== undefined) - return segment.latestMetas; - const metas = new Map(); - for (const [key, history] of segment.histories) { - const meta = history.at(-1); - if (meta !== undefined) - metas.set(key, meta); - } - segment.latestMetas = metas; - return metas; - } -} -function cloneHistories(histories) { - return new Map([...histories].map(([key, history]) => [key, [...history]])); -} -function normalizeRuns(runs) { - // Single-run input is the common case and is already normalized; returning - // it as-is skips the filter/map/sort chain. Callers only read the result. - if (runs.length === 0) - return []; - if (runs.length === 1) { - const run = runs[0]; - return run.length > 0 ? [run] : []; - } - const sorted = runs - .filter((run) => run.length > 0) - .map((run) => ({ start: { ...run.start }, length: run.length })) - .sort((left, right) => left.start.peer < right.start.peer - ? -1 - : left.start.peer > right.start.peer - ? 1 - : left.start.counter - right.start.counter); - const merged = []; - for (const run of sorted) { - const previous = merged.at(-1); - const end = run.start.counter + run.length; - if (previous !== undefined && - previous.start.peer === run.start.peer && - run.start.counter <= previous.start.counter + previous.length) { - previous.length = - Math.max(previous.start.counter + previous.length, end) - previous.start.counter; - } - else { - merged.push(run); - } - } - return merged; -} -function compareMeta(left, right) { - return (left.lamport - right.lamport || - (left.startId.peer < right.startId.peer - ? -1 - : left.startId.peer > right.startId.peer - ? 1 - : 0)); -} -function lowerBoundMeta(history, meta) { - let low = 0; - let high = history.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (compareMeta(history[middle], meta) < 0) - low = middle + 1; - else - high = middle; - } - return low; -} -function latestIncluded(history, version) { - if (history === undefined) - return undefined; - if (version === undefined) - return history.at(-1); - for (let index = history.length - 1; index >= 0; index -= 1) { - const meta = history[index]; - if (meta.startId.counter < (version.get(meta.startId.peer) ?? 0)) - return meta; - } - return undefined; -} -function sameMeta(left, right) { - return (left?.startId.peer === right.startId.peer && - left.startId.counter === right.startId.counter); -} diff --git a/loro-js/src/runtime/types.js b/loro-js/src/runtime/types.js deleted file mode 100644 index cb0ff5c3b..000000000 --- a/loro-js/src/runtime/types.js +++ /dev/null @@ -1 +0,0 @@ -export {}; diff --git a/loro-js/src/runtime/undo.js b/loro-js/src/runtime/undo.js deleted file mode 100644 index d69448098..000000000 --- a/loro-js/src/runtime/undo.js +++ /dev/null @@ -1,274 +0,0 @@ -class UndoDeque { - // Array + head offset: capacity is capped by maxUndoSteps, so a Map of - // slot indexes only costs extra entries per push/pop. - #items = []; - #start = 0; - get length() { - return this.#items.length - this.#start; - } - push(item) { - this.#items.push(item); - } - pop() { - if (this.length === 0) - return undefined; - const item = this.#items.pop(); - if (this.length === 0) - this.clear(); - return item; - } - peek() { - return this.length === 0 ? undefined : this.#items[this.#items.length - 1]; - } - trimFront(length) { - while (this.length > length) - this.#start += 1; - if (this.#start === 0) - return; - if (this.#start >= this.#items.length) { - this.clear(); - } - else if (this.#start >= 32 && this.#start * 2 >= this.#items.length) { - // Release the trimmed slots once they dominate the backing array. - this.#items.copyWithin(0, this.#start); - this.#items.length -= this.#start; - this.#start = 0; - } - } - clear() { - this.#items.length = 0; - this.#start = 0; - } -} -const EMPTY_META = { value: null, cursors: [] }; -export class UndoManager { - #doc; - #peer; - #undo = new UndoDeque(); - #redo = []; - #excludeOriginPrefixes = new Set(); - #remoteTargets = new Set(); - #mergeInterval; - #maxUndoSteps; - #onPush; - #onPop; - #applying = false; - #groupDepth = 0; - #unsubscribe; - constructor(doc, config = {}) { - this.#doc = doc; - this.#peer = doc.peerIdStr; - this.#mergeInterval = config.mergeInterval ?? 1000; - this.#maxUndoSteps = config.maxUndoSteps ?? 100; - this.#onPush = config.onPush; - this.#onPop = config.onPop; - for (const prefix of config.excludeOriginPrefixes ?? []) { - this.#excludeOriginPrefixes.add(prefix); - } - this.#unsubscribe = doc.subscribe((event) => this.#record(event)); - } - free() { - this.destroy(); - } - undo() { - const item = this.#undo.pop(); - if (item === undefined) - return false; - try { - const redo = this.#invert(item, true); - if (redo !== undefined) - this.#redo.push(redo); - } - catch (error) { - this.#undo.push(item); - throw error; - } - return true; - } - redo() { - const item = this.#redo.pop(); - if (item === undefined) - return false; - try { - const undo = this.#invert(item, false); - if (undo !== undefined) - this.#pushUndo(undo, false); - } - catch (error) { - this.#redo.push(item); - throw error; - } - return true; - } - peer() { - return this.#peer; - } - groupStart() { - this.#groupDepth += 1; - } - groupEnd() { - if (this.#groupDepth > 0) - this.#groupDepth -= 1; - } - canUndo() { - return this.#undo.length > 0; - } - canRedo() { - return this.#redo.length > 0; - } - topUndoValue() { - return this.#undo.peek()?.meta.value; - } - topRedoValue() { - return this.#redo.at(-1)?.meta.value; - } - setMaxUndoSteps(steps) { - if (!Number.isSafeInteger(steps) || steps < 0) { - throw new RangeError("max undo steps must be a nonnegative integer"); - } - this.#maxUndoSteps = steps; - this.#trimUndo(); - } - setMergeInterval(interval) { - if (!Number.isFinite(interval) || interval < 0) { - throw new RangeError("merge interval must be nonnegative"); - } - this.#mergeInterval = interval; - } - addExcludeOriginPrefix(prefix) { - this.#excludeOriginPrefixes.add(prefix); - } - setOnPush(callback) { - this.#onPush = callback; - } - setOnPop(callback) { - this.#onPop = callback; - } - clear() { - this.#undo.clear(); - this.#redo.length = 0; - this.#remoteTargets.clear(); - } - clearUndo() { - this.#undo.clear(); - } - clearRedo() { - this.#redo.length = 0; - } - destroy() { - this.#unsubscribe(); - this.clear(); - } - #record(event) { - if (this.#applying) - return; - const targets = new Set(); - for (const { target } of event.events) - targets.add(target); - if (event.by === "checkout") { - this.clear(); - return; - } - if (event.by === "import") { - for (const target of targets) - this.#remoteTargets.add(target); - return; - } - if (event.origin !== undefined && this.#excludedOrigin(event.origin)) { - for (const target of targets) - this.#remoteTargets.add(target); - return; - } - const spans = this.#doc.findIdSpansBetween(event.from, event.to).forward; - const span = spans.find(({ peer }) => peer === this.#doc.peerIdStr) ?? - (spans.length === 1 ? spans[0] : undefined); - if (span === undefined || span.length === 0) - return; - this.#peer = span.peer; - const now = Date.now(); - const item = { - peer: span.peer, - range: { start: span.counter, end: span.counter + span.length }, - meta: this.#onPush?.(true, { start: span.counter, end: span.counter + span.length }, event) ?? EMPTY_META, - timestamp: now, - targets, - }; - const previous = this.#undo.peek(); - let conflictsWithRemote = false; - if (previous !== undefined) { - for (const target of previous.targets) { - if (this.#remoteTargets.has(target)) { - conflictsWithRemote = true; - break; - } - } - } - const merge = previous !== undefined && - previous.peer === item.peer && - previous.range.end === item.range.start && - !conflictsWithRemote && - (this.#groupDepth > 0 || - (this.#mergeInterval > 0 && now - previous.timestamp < this.#mergeInterval)); - if (merge) { - previous.range = { start: previous.range.start, end: item.range.end }; - previous.meta = item.meta; - previous.timestamp = now; - for (const target of item.targets) - previous.targets.add(target); - } - else { - this.#pushUndo(item, false); - } - this.#redo.length = 0; - this.#remoteTargets.clear(); - } - #invert(item, isUndo) { - const before = this.#doc.frontiers(); - this.#applying = true; - try { - this.#doc._undoIdSpan(item.peer, item.range); - this.#doc.commit({ origin: isUndo ? "undo" : "redo" }); - const after = this.#doc.frontiers(); - const spans = this.#doc.findIdSpansBetween(before, after).forward; - const span = spans.find(({ peer }) => peer === this.#doc.peerIdStr) ?? - (spans.length === 1 ? spans[0] : undefined); - const poppedMeta = { - value: item.meta.value, - cursors: this.#doc._transformUndoCursors(item.meta.cursors), - }; - this.#onPop?.(isUndo, poppedMeta, item.range); - if (span === undefined || span.length === 0) - return undefined; - const range = { start: span.counter, end: span.counter + span.length }; - this.#peer = span.peer; - return { - peer: span.peer, - range, - meta: this.#onPush?.(!isUndo, range) ?? EMPTY_META, - timestamp: Date.now(), - targets: new Set(item.targets), - }; - } - finally { - this.#applying = false; - } - } - #pushUndo(item, clearRedo) { - this.#undo.push(item); - this.#trimUndo(); - if (clearRedo) - this.#redo.length = 0; - } - #excludedOrigin(origin) { - if (this.#excludeOriginPrefixes.size === 0) - return false; - for (const prefix of this.#excludeOriginPrefixes) { - if (origin.startsWith(prefix)) - return true; - } - return false; - } - #trimUndo() { - this.#undo.trimFront(this.#maxUndoSteps); - } -} diff --git a/loro-js/src/runtime/version-vector.js b/loro-js/src/runtime/version-vector.js deleted file mode 100644 index 9acbbbb02..000000000 --- a/loro-js/src/runtime/version-vector.js +++ /dev/null @@ -1,112 +0,0 @@ -import { decodePostcardVersionVector, encodePostcardVersionVector, } from "../codec/version"; -import { formatOpId, parseOpId, parsePeerId, peerIdToString } from "./ids"; -export class VersionVector { - #values = new Map(); - constructor(input) { - if (input == null) - return; - if (input instanceof Uint8Array) { - for (const id of decodePostcardVersionVector(input)) - this.#values.set(id.peer, id.counter); - return; - } - if (input instanceof VersionVector) { - for (const [peer, counter] of input.#values) - this.#values.set(peer, counter); - return; - } - const entries = input instanceof Map ? input.entries() : Object.entries(input); - for (const [peer, counter] of entries) - this.set(peer, counter); - } - free() { } - static parseJSON(input) { - return new VersionVector(input); - } - static decode(bytes) { - return new VersionVector(bytes); - } - encode() { - return encodePostcardVersionVector(this.codecEntries()); - } - toJSON() { - return new Map([...this.#values].map(([peer, counter]) => [peerIdToString(peer), counter])); - } - get(peer) { - return this.#values.get(parsePeerId(peer)); - } - compare(other) { - let less = false; - let greater = false; - for (const [peer, left] of this.#values) { - const right = other.#values.get(peer) ?? 0; - if (left < right) - less = true; - else if (left > right) - greater = true; - if (less && greater) - return undefined; - } - // Peers present only in the other vector compare as 0 on this side. - for (const [peer, right] of other.#values) { - if (this.#values.has(peer)) - continue; - if (right > 0) - less = true; - else if (right < 0) - greater = true; - if (less && greater) - return undefined; - } - return less ? -1 : greater ? 1 : 0; - } - setEnd(id) { - const parsed = parseOpId(id); - this.set(parsed.peer, parsed.counter); - } - setLast(id) { - const parsed = parseOpId(id); - this.set(parsed.peer, parsed.counter + 1); - } - remove(peer) { - this.#values.delete(parsePeerId(peer)); - } - length() { - return this.#values.size; - } - clone() { - return new VersionVector(this); - } - codecEntries() { - const entries = this._codecEntriesUnsorted(); - entries.sort((left, right) => left.peer < right.peer ? -1 : left.peer > right.peer ? 1 : 0); - return entries; - } - _codecEntriesUnsorted() { - const entries = []; - for (const [peer, counter] of this.#values) { - if (counter > 0) - entries.push({ peer, counter }); - } - return entries; - } - publicEntries() { - return this.codecEntries().map(formatOpId); - } - set(peer, counter) { - if (!Number.isSafeInteger(counter) || counter < 0 || counter > 2147483647) { - throw new RangeError(`version counter is out of range: ${counter}`); - } - const parsed = parsePeerId(peer); - if (counter === 0) - this.#values.delete(parsed); - else - this.#values.set(parsed, counter); - } - merge(other) { - for (const [peer, counter] of other.#values) { - if (counter > (this.#values.get(peer) ?? 0)) - this.#values.set(peer, counter); - } - } -} diff --git a/loro-js/tests/change-block-codec.test.js b/loro-js/tests/change-block-codec.test.js deleted file mode 100644 index beb185ebf..000000000 --- a/loro-js/tests/change-block-codec.test.js +++ /dev/null @@ -1,133 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { ContainerType, decodeChangeBlock, encodeChangeBlock, validateChangeBlock, } from "../src/codec/index"; -describe("semantic change block codec", () => { - test("round trips a map insert block", () => { - const container = { - kind: "root", - name: "root", - containerType: ContainerType.Map, - }; - const block = { - peers: [0x0102030405060708n], - keys: ["root", "a"], - containers: [container], - positions: [], - changes: [ - { - id: { peer: 0x0102030405060708n, counter: 10 }, - timestamp: 1234n, - dependencies: [], - lamport: 100, - message: undefined, - operations: [ - { - container, - counter: 10, - length: 1, - content: { - type: "map-insert", - key: "a", - value: { type: "i64", value: 10n }, - }, - }, - ], - }, - ], - }; - const encoded = encodeChangeBlock(block); - expect(decodeChangeBlock(encoded)).toEqual(block); - expect(validateChangeBlock(encoded)).toEqual({ - peer: 0x0102030405060708n, - counterStart: 10, - counterEnd: 11, - }); - }); - test("round trips deletes, movable-list operations and tree operations", () => { - const peer = 7n; - const list = { - kind: "root", - name: "list", - containerType: ContainerType.MovableList, - }; - const tree = { - kind: "root", - name: "tree", - containerType: ContainerType.Tree, - }; - const block = { - peers: [peer], - keys: ["list", "tree"], - containers: [list, tree], - positions: [Uint8Array.of(1, 2)], - changes: [ - { - id: { peer, counter: 0 }, - timestamp: 0n, - dependencies: [], - lamport: 0, - message: "ops", - operations: [ - { - container: list, - counter: 0, - length: 1, - content: { - type: "movable-list-insert", - position: 0, - values: [{ type: "string", value: "x" }], - }, - }, - { - container: list, - counter: 1, - length: 1, - content: { - type: "movable-list-delete", - position: 0, - length: 1n, - startId: { peer, counter: 0 }, - }, - }, - { - container: tree, - counter: 2, - length: 1, - content: { - type: "tree-create", - subject: { peer, counter: 2 }, - parent: undefined, - position: Uint8Array.of(1, 2), - }, - }, - { - container: tree, - counter: 3, - length: 1, - content: { - type: "tree-create", - subject: { peer, counter: 3 }, - parent: { peer, counter: 2 }, - position: Uint8Array.of(1, 2), - }, - }, - { - container: tree, - counter: 4, - length: 1, - content: { type: "tree-delete", subject: { peer, counter: 2 } }, - }, - ], - }, - ], - }; - const encoded = encodeChangeBlock(block); - const decoded = decodeChangeBlock(encoded); - expect(decoded.changes).toEqual(block.changes); - expect(decoded.peers).toEqual(expect.arrayContaining([peer, 0xffffffffffffffffn])); - expect(validateChangeBlock(encoded)).toEqual({ - peer, - counterStart: 0, - counterEnd: 5, - }); - }); -}); diff --git a/loro-js/tests/change-block-tables.test.js b/loro-js/tests/change-block-tables.test.js deleted file mode 100644 index f19981f95..000000000 --- a/loro-js/tests/change-block-tables.test.js +++ /dev/null @@ -1,69 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { ContainerType, decodeChangeKeys, decodeChangesHeader, decodeChangesMetadata, decodeContainerArena, decodeDeleteStartIds, decodeEncodedOperations, decodePositionArena, encodeChangeKeys, encodeChangesHeader, encodeChangesMetadata, encodeContainerArena, encodeDeleteStartIds, encodeEncodedOperations, encodePositionArena, } from "../src/codec/index"; -describe("change block tables", () => { - test("round trips a change header and metadata", () => { - const header = { - peer: 7n, - peers: [7n, 9n], - counters: [10, 12, 15], - lengths: [2, 3], - lamports: [20, 23], - dependencies: [ - [{ peer: 9n, counter: 3 }], - [ - { peer: 7n, counter: 11 }, - { peer: 9n, counter: 4 }, - ], - ], - }; - const encodedHeader = encodeChangesHeader(header); - expect(decodeChangesHeader(encodedHeader, { - changeCount: 2, - counterStart: 10, - counterLength: 5, - lamportStart: 20, - lamportLength: 6, - })).toEqual(header); - const metadata = { - timestamps: [-1n, 42n], - commitMessages: [undefined, "commit 😀"], - }; - expect(decodeChangesMetadata(encodeChangesMetadata(metadata), 2)).toEqual(metadata); - }); - test("round trips keys and the container arena", () => { - const keys = ["map", "key 😀"]; - const containers = [ - { kind: "root", name: "map", containerType: ContainerType.Map }, - { - kind: "normal", - peer: 9n, - counter: 4, - containerType: ContainerType.Text, - }, - ]; - expect(decodeChangeKeys(encodeChangeKeys(keys))).toEqual(keys); - expect(decodeContainerArena(encodeContainerArena(containers, [7n, 9n], keys), [7n, 9n], keys)).toEqual(containers); - }); - test("round trips operation and delete tables", () => { - const operations = [ - { containerIndex: 0, property: 2, valueType: 11, length: 1 }, - { containerIndex: 1, property: -1, valueType: 9, length: 3 }, - ]; - const deletes = [ - { peerIndex: 1n, counter: 4, length: -3n }, - ]; - expect(decodeEncodedOperations(encodeEncodedOperations(operations))).toEqual(operations); - expect(decodeDeleteStartIds(encodeDeleteStartIds(deletes))).toEqual(deletes); - }); - test("round trips the prefix-compressed position arena", () => { - const positions = [ - Uint8Array.of(1, 2, 3), - Uint8Array.of(1, 2, 4), - Uint8Array.of(1, 2, 4, 255), - Uint8Array.of(1), - new Uint8Array(), - ]; - expect(decodePositionArena(encodePositionArena(positions))).toEqual(positions); - expect(decodePositionArena(new Uint8Array())).toEqual([]); - }); -}); diff --git a/loro-js/tests/change-value.test.js b/loro-js/tests/change-value.test.js deleted file mode 100644 index 39f2fd03a..000000000 --- a/loro-js/tests/change-value.test.js +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { bytesToHex, decodeChangeValue, encodeChangeValue } from "../src/codec/index"; -const bytes = (hex) => Uint8Array.from(hex.match(/../g) ?? [], (byte) => Number.parseInt(byte, 16)); -describe("change value codec", () => { - test.each([ - ["00", { type: "null" }], - ["01", { type: "bool", value: true }], - ["02", { type: "bool", value: false }], - ["037f", { type: "i64", value: -1n }], - ["0a7f", { type: "delta-int", value: -1 }], - ["05026869", { type: "string", value: "hi" }], - ["0603010203", { type: "binary", value: Uint8Array.of(1, 2, 3) }], - ["07ac02", { type: "container-index", value: 300n }], - ["08", { type: "delete-once" }], - ["09", { type: "delete-sequence" }], - ["043ff0000000000000", { type: "double", value: 1 }], - [ - "0b07020301050178", - { - type: "loro-value", - value: { - type: "list", - value: [ - { type: "i64", value: 1n }, - { type: "string", value: "x" }, - ], - }, - }, - ], - [ - "0b08010000", - { - type: "loro-value", - value: { type: "map", value: [[0n, { type: "null" }]] }, - }, - ], - [ - "0c84010001", - { - type: "mark-start", - info: 0x84, - length: 1n, - keyIndex: 0n, - value: { type: "bool", value: true }, - }, - ], - [ - "0d010102", - { - type: "tree-move", - targetIndex: 1n, - parentIsNull: true, - position: 2n, - parentIndex: undefined, - }, - ], - [ - "0d01000203", - { - type: "tree-move", - targetIndex: 1n, - parentIsNull: false, - position: 2n, - parentIndex: 3n, - }, - ], - ["0e010203", { type: "list-move", from: 1n, fromPeerIndex: 2n, lamport: 3n }], - [ - "0f072a0301", - { - type: "list-set", - peerIndex: 7n, - lamport: 42, - value: { type: "i64", value: 1n }, - }, - ], - [ - "1001020301", - { - type: "raw-tree-move", - subjectPeerIndex: 1n, - subjectCounter: 2, - positionIndex: 3n, - parentIsNull: true, - parentPeerIndex: 0n, - parentCounter: 0, - }, - ], - [ - "10010203000405", - { - type: "raw-tree-move", - subjectPeerIndex: 1n, - subjectCounter: 2, - positionIndex: 3n, - parentIsNull: false, - parentPeerIndex: 4n, - parentCounter: 5, - }, - ], - ["910158", { type: "future", tag: 0x91, data: Uint8Array.of(0x58) }], - ])("matches the known bytes %s", (hex, value) => { - expect(decodeChangeValue(bytes(hex))).toEqual(value); - expect(bytesToHex(encodeChangeValue(value))).toBe(hex); - }); -}); diff --git a/loro-js/tests/document.test.js b/loro-js/tests/document.test.js deleted file mode 100644 index 8d9eeb7d1..000000000 --- a/loro-js/tests/document.test.js +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { EncodeMode, LoroDecodeError, bytesToHex, decodeDocument, decodeFastSnapshot, decodeFastUpdates, encodeDocument, encodeFastSnapshot, encodeFastUpdates, } from "../src/codec/index"; -describe("Loro document envelope", () => { - test("covers mode and body with the checksum", () => { - const encoded = encodeDocument(EncodeMode.FastUpdates, Uint8Array.of(1, 2, 3)); - expect(bytesToHex(encoded.subarray(0, 4))).toBe("6c6f726f"); - expect(encoded.subarray(4, 16)).toEqual(new Uint8Array(12)); - expect(encoded.subarray(20, 22)).toEqual(Uint8Array.of(0, 4)); - expect(decodeDocument(encoded)).toMatchObject({ mode: EncodeMode.FastUpdates }); - const corrupted = encoded.slice(); - corrupted[20] = corrupted[20] ^ 1; - expect(() => decodeDocument(corrupted)).toThrow(LoroDecodeError); - }); -}); -describe("FastSnapshot", () => { - test("round trips all three exact sections", () => { - const snapshot = { - oplog: Uint8Array.of(1, 2), - state: Uint8Array.of(3), - shallowRootState: Uint8Array.of(4, 5, 6), - }; - expect(decodeFastSnapshot(encodeFastSnapshot(snapshot))).toEqual(snapshot); - }); - test("rejects trailing section bytes", () => { - const encoded = encodeFastSnapshot({ - oplog: new Uint8Array(), - state: new Uint8Array(), - shallowRootState: new Uint8Array(), - }); - const document = decodeDocument(encoded); - const withTrailing = encodeDocument(EncodeMode.FastSnapshot, new Uint8Array([...document.body, 0])); - expect(() => decodeFastSnapshot(withTrailing)).toThrow("trailing FastSnapshot bytes"); - }); -}); -describe("FastUpdates", () => { - test("round trips length-prefixed postcard blocks", () => { - const blocks = [new Uint8Array(), Uint8Array.of(1), new Uint8Array(300).fill(7)]; - expect(decodeFastUpdates(encodeFastUpdates(blocks))).toEqual(blocks); - }); -}); diff --git a/loro-js/tests/event-diff.test.js b/loro-js/tests/event-diff.test.js deleted file mode 100644 index 56a715ce5..000000000 --- a/loro-js/tests/event-diff.test.js +++ /dev/null @@ -1,290 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { SequenceEventDiff } from "../src/runtime/event-diff"; -import { LoroDoc } from "../src/runtime"; -describe("incremental event diffs", () => { - test("composes randomized text edits against a string model", () => { - const original = "abcdefghijklmnopqrstuvwxyz"; - const diff = new SequenceEventDiff("text", original.length); - let current = original; - let random = 1367130551; - const next = () => { - random ^= random << 13; - random ^= random >>> 17; - random ^= random << 5; - return random >>> 0; - }; - for (let step = 0; step < 1000; step += 1) { - if (current.length === 0 || (next() & 1) === 0) { - const position = next() % (current.length + 1); - const value = String.fromCharCode(65 + (next() % 26)); - diff.insertText(position, value); - current = current.slice(0, position) + value + current.slice(position); - } - else { - const position = next() % current.length; - const length = 1 + (next() % Math.min(4, current.length - position)); - diff.delete(position, length); - current = current.slice(0, position) + current.slice(position + length); - } - } - const result = diff.toDiff(); - expect(result.type).toBe("text"); - expect(applyTextDelta(original, result.diff)).toBe(current); - }); - test("composes randomized list edits against an array model", () => { - const original = Array.from({ length: 32 }, (_, index) => index); - const diff = new SequenceEventDiff("list", original.length); - let current = [...original]; - let random = 429261588; - const next = () => { - random ^= random << 13; - random ^= random >>> 17; - random ^= random << 5; - return random >>> 0; - }; - for (let step = 0; step < 1000; step += 1) { - if (current.length === 0 || (next() & 1) === 0) { - const position = next() % (current.length + 1); - const value = `v${step}`; - diff.insertList(position, [value]); - current.splice(position, 0, value); - } - else { - const position = next() % current.length; - const length = 1 + (next() % Math.min(4, current.length - position)); - diff.delete(position, length); - current.splice(position, length); - } - } - const result = diff.toDiff(); - expect(result.type).toBe("list"); - expect(applyListDelta(original, result.diff)).toEqual(current); - }); - test("keeps text formatting as an attributed retain", () => { - const diff = new SequenceEventDiff("text", 6); - diff.formatText(1, 3, "bold", true); - expect(diff.toDiff()).toEqual({ - type: "text", - diff: [{ retain: 1 }, { retain: 3, attributes: { bold: true } }], - }); - }); - test("emits operation-composed local diffs for every indexed container", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const text = source.getText("text"); - const list = source.getList("list"); - const movable = source.getMovableList("movable"); - const map = source.getMap("map"); - const counter = source.getCounter("counter"); - text.insert(0, "abcdef"); - list.push("a"); - list.push("b"); - list.push("c"); - movable.push("a"); - movable.push("b"); - movable.push("c"); - map.set("value", 1); - counter.increment(2); - source.commit(); - const target = new LoroDoc(); - target.setPeerId(2); - target.getText("text").insert(0, "abcdef"); - target.getList("list").push("a"); - target.getList("list").push("b"); - target.getList("list").push("c"); - target.getMovableList("movable").push("a"); - target.getMovableList("movable").push("b"); - target.getMovableList("movable").push("c"); - target.getMap("map").set("value", 1); - target.getCounter("counter").increment(2); - target.commit(); - let batch; - source.subscribe((event) => { - batch = event; - }); - text.delete(1, 2); - text.insert(2, "XY"); - text.mark({ start: 1, end: 4 }, "bold", true); - list.delete(1, 1); - list.insert(1, "B"); - movable.move(0, 2); - movable.set(1, "B"); - map.set("value", 2); - map.set("transient", true); - map.delete("transient"); - counter.increment(3); - counter.decrement(1); - source.commit(); - expect(batch).toBeDefined(); - target.applyDiff(batch.events.map(({ target: id, diff: eventDiff }) => [id, eventDiff])); - expect(target.toJSON()).toEqual(source.toJSON()); - expect(target.getText("text").toDelta()).toEqual(text.toDelta()); - expect(batch.events.find(({ target: id }) => id === map.id)?.diff).toEqual({ - type: "map", - updated: { value: 2 }, - }); - expect(batch.events.find(({ target: id }) => id === counter.id)?.diff).toEqual({ - type: "counter", - increment: 2, - }); - }); - test("records imported concurrent edits at their actual visible positions", () => { - const base = new LoroDoc(); - base.setPeerId(1); - base.getText("text").insert(0, "abcd"); - for (const value of ["a", "b", "c", "d"]) - base.getList("list").push(value); - for (const value of ["a", "b", "c"]) { - base.getMovableList("movable").push(value); - } - base.getMap("map").set("value", 1); - base.getCounter("counter").increment(1); - base.commit(); - const baseVersion = base.oplogVersion(); - const left = base.fork(); - left.setPeerId(2); - left.getText("text").insert(2, "L"); - left.getText("text").delete(0, 1); - left.getText("text").mark({ start: 0, end: 3 }, "bold", true); - left.getList("list").insert(2, "L"); - left.getList("list").delete(0, 1); - left.getMovableList("movable").move(0, 2); - left.getMovableList("movable").set(1, "B"); - left.getMap("map").set("value", 2); - left.getCounter("counter").increment(2); - left.commit(); - const update = left.export({ mode: "update", from: baseVersion }); - const right = base.fork(); - right.setPeerId(3); - right.getText("text").insert(2, "R"); - right.getList("list").insert(2, "R"); - right.getMovableList("movable").insert(1, "R"); - right.commit(); - const mirror = right.fork(); - let batch; - right.subscribe((event) => { - batch = event; - }); - right.import(update); - expect(batch).toBeDefined(); - mirror.applyDiff(batch.events.map(({ target: id, diff: eventDiff }) => [id, eventDiff])); - expect(mirror.toJSON()).toEqual(right.toJSON()); - expect(mirror.getText("text").toDelta()).toEqual(right.getText("text").toDelta()); - }); - test("emits only directly changed tree nodes", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const tree = doc.getTree("tree"); - tree.createNode(); - tree.createNode(); - tree.createNode(); - doc.commit(); - let batch; - doc.subscribe((event) => { - batch = event; - }); - const created = tree.createNode(undefined, 0); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ - type: "tree", - diff: [ - { - target: created.id, - action: "create", - parent: undefined, - index: 0, - fractionalIndex: created.fractionalIndex(), - }, - ], - }); - const moved = tree.roots()[2]; - const oldIndex = moved.index(); - moved.move(undefined, 0); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ - type: "tree", - diff: [ - { - target: moved.id, - action: "move", - parent: undefined, - index: 0, - fractionalIndex: moved.fractionalIndex(), - oldParent: undefined, - oldIndex, - }, - ], - }); - }); - test("reports local tree reindexing during a concurrent import", () => { - const incoming = new LoroDoc(); - incoming.setPeerId(1); - const incomingNode = incoming.getTree("tree").createNode(); - incoming.commit(); - const target = new LoroDoc(); - target.setPeerId(2); - const localNode = target.getTree("tree").createNode(); - target.commit(); - let batch; - target.subscribe((event) => { - batch = event; - }); - target.import(incoming.export({ mode: "update" })); - expect(batch?.events[0]?.diff).toEqual({ - type: "tree", - diff: [ - { - target: localNode.id, - action: "move", - parent: undefined, - index: 1, - fractionalIndex: localNode.fractionalIndex(), - oldParent: undefined, - oldIndex: 0, - }, - { - target: incomingNode.id, - action: "create", - parent: undefined, - index: 0, - fractionalIndex: incomingNode.fractionalIndex(), - }, - ], - }); - }); -}); -function applyTextDelta(original, delta) { - let cursor = 0; - let output = ""; - for (const operation of delta) { - if ("retain" in operation) { - output += original.slice(cursor, cursor + operation.retain); - cursor += operation.retain; - } - else if ("delete" in operation) { - cursor += operation.delete; - } - else { - output += operation.insert; - } - } - return output + original.slice(cursor); -} -function applyListDelta(original, delta) { - let cursor = 0; - const output = []; - for (const operation of delta) { - if ("retain" in operation) { - output.push(...original.slice(cursor, cursor + operation.retain)); - cursor += operation.retain; - } - else if ("delete" in operation) { - cursor += operation.delete; - } - else { - output.push(...operation.insert); - } - } - output.push(...original.slice(cursor)); - return output; -} diff --git a/loro-js/tests/fugue-index.test.js b/loro-js/tests/fugue-index.test.js deleted file mode 100644 index c616b5bab..000000000 --- a/loro-js/tests/fugue-index.test.js +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { LoroText } from "../src/index"; -import { SequenceIndex } from "../src/runtime/sequence-index"; -describe("Fugue origin index", () => { - test("does not probe every descendant in a concurrent text run", () => { - const probeCount = (length) => { - const text = new LoroText(); - text._insertFugue(0, "a".repeat(length), { peer: 1n, counter: 0 }, 0, new Map()); - const atPhysical = vi.spyOn(SequenceIndex.prototype, "atPhysicalRaw"); - text._insertFugue(0, "b", { peer: 2n, counter: 0 }, 0, new Map()); - const probes = atPhysical.mock.calls.length; - atPhysical.mockRestore(); - expect(text.toString()).toBe(`${"a".repeat(length)}b`); - return probes; - }; - const shortRunProbes = probeCount(128); - const longRunProbes = probeCount(16384); - expect(longRunProbes).toBeLessThanOrEqual(shortRunProbes + 2); - expect(longRunProbes).toBeLessThan(8); - }); -}); diff --git a/loro-js/tests/indexes.test.js b/loro-js/tests/indexes.test.js deleted file mode 100644 index 1ff7fd377..000000000 --- a/loro-js/tests/indexes.test.js +++ /dev/null @@ -1,620 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { OrderedIndex } from "../src/runtime/ordered-index"; -import { SequenceDeletionIndex } from "../src/runtime/sequence-deletion-index"; -import { SequenceIndex } from "../src/runtime/sequence-index"; -import { TextStyleIndex } from "../src/runtime/text-style-index"; -describe("runtime indexes", () => { - test("maintains sequence ranks, metrics, and id lookups", () => { - const sequence = new SequenceIndex((element) => ({ - utf16: element.value.length, - utf8: new TextEncoder().encode(element.value).length, - })); - const physical = []; - let counter = 0; - let random = 305419896; - const nextRandom = () => { - random ^= random << 13; - random ^= random >>> 17; - random ^= random << 5; - return random >>> 0; - }; - const visible = () => physical.filter((element) => !element.deleted); - for (let step = 0; step < 2000; step += 1) { - const current = visible(); - const operation = current.length === 0 ? 0 : nextRandom() % 3; - if (operation === 0) { - const position = nextRandom() % (current.length + 1); - const value = ["a", "文", "🙂"][nextRandom() % 3]; - const element = { - id: { peer: 1n, counter: counter++ }, - deleted: false, - deletedBy: [], - value, - }; - const physicalPosition = position === current.length - ? physical.length - : physical.indexOf(current[position]); - physical.splice(physicalPosition, 0, element); - sequence.insertAtVisible(position, [element]); - } - else if (operation === 1) { - const element = current[nextRandom() % current.length]; - sequence.setDeleted(element, true); - } - else { - const from = nextRandom() % current.length; - const to = nextRandom() % current.length; - if (from !== to) { - const element = current[from]; - const physicalPosition = physical.indexOf(element); - physical.splice(physicalPosition, 1); - const afterRemoval = visible(); - const destination = to >= afterRemoval.length - ? physical.length - : physical.indexOf(afterRemoval[to]); - physical.splice(destination, 0, element); - sequence.moveVisible(from, to); - } - } - if (step % 25 !== 0) - continue; - const expectedVisible = visible(); - expect(sequence.all()).toEqual(physical); - expect(sequence.visible()).toEqual(expectedVisible); - expect(sequence.allLength).toBe(physical.length); - expect(sequence.visibleLength).toBe(expectedVisible.length); - expect(sequence.visibleUtf16Length).toBe(expectedVisible.reduce((length, element) => length + element.value.length, 0)); - expect(sequence.visibleUtf8Length).toBe(new TextEncoder().encode(expectedVisible.map(({ value }) => value).join("")) - .length); - for (const [index, element] of physical.entries()) { - expect(sequence.atPhysical(index)).toBe(element); - expect(sequence.physicalIndexOf(element)).toBe(index); - expect(sequence.findById(element.id)).toBe(element); - } - for (const [index, element] of expectedVisible.entries()) { - expect(sequence.atVisible(index)).toBe(element); - expect(sequence.visibleIndexOf(element)).toBe(index); - } - const start = expectedVisible.length >>> 2; - const end = expectedVisible.length - start; - expect(sequence.visibleRange(start, end)).toEqual(expectedVisible.slice(start, end)); - expect(sequence.visibleIdRuns(start, end)).toEqual(compressIdRuns(expectedVisible.slice(start, end))); - } - }); - test("stops visible range predicates after the first match", () => { - const sequence = new SequenceIndex(); - sequence.insertAtVisible(0, Array.from({ length: 96 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: counter % 7 === 0, - deletedBy: [], - value: String(counter), - }))); - let visits = 0; - expect(sequence.someVisibleRange(10, 80, (element) => { - visits += 1; - return element.value === "18"; - })).toBe(true); - expect(visits).toBeLessThan(10); - visits = 0; - expect(sequence.someVisibleRange(10, 20, () => { - visits += 1; - return false; - })).toBe(false); - expect(visits).toBe(10); - }); - test("visits raw visible storage ranges and locates insertion neighbor ids", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 6 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtVisible(0, elements); - sequence.setDeleted(elements[1], true); - sequence.setDeleted(elements[4], true); - const ranges = []; - sequence.forEachVisibleStorageRange((_storage, start, end) => { - ranges.push({ start, end }); - }); - expect(ranges).toEqual([ - { start: 0, end: 1 }, - { start: 2, end: 4 }, - { start: 5, end: 6 }, - ]); - const context = { - leftPeer: undefined, - leftCounter: 0, - startIndex: 0, - rightPeer: undefined, - rightCounter: 0, - }; - const locate = (position) => ({ - ...sequence.visibleInsertionIdContext(position, context), - }); - expect(locate(0)).toEqual({ - leftPeer: undefined, - leftCounter: 0, - startIndex: 0, - rightPeer: 1n, - rightCounter: 0, - }); - expect(locate(1)).toEqual({ - leftPeer: 1n, - leftCounter: 0, - startIndex: 1, - rightPeer: 1n, - rightCounter: 1, - }); - expect(locate(2)).toEqual({ - leftPeer: 1n, - leftCounter: 2, - startIndex: 3, - rightPeer: 1n, - rightCounter: 3, - }); - expect(locate(4)).toEqual({ - leftPeer: 1n, - leftCounter: 5, - startIndex: 6, - rightPeer: undefined, - rightCounter: 0, - }); - expect(() => sequence.loadValidatedSpans([])).toThrow(/validated sequence spans require an empty index/u); - }); - test("maps id runs to visible metric ranges without crossing inserts", () => { - const sequence = new SequenceIndex((element) => ({ - utf16: element.value.length, - utf8: new TextEncoder().encode(element.value).length, - })); - const original = ["a", "🙂", "b", "c", "d"].map((value, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value, - })); - sequence.insertAtVisible(0, original); - sequence.insertAtVisible(2, [ - { - id: { peer: 2n, counter: 0 }, - deleted: false, - deletedBy: [], - value: "X", - }, - ]); - sequence.setDeleted(original[3], true); - expect(sequence.visibleMetricRangesForIdRuns([{ start: { peer: 1n, counter: 0 }, length: 5 }], "utf16")).toEqual([ - { start: 0, end: 3 }, - { start: 4, end: 6 }, - ]); - }); - test("maintains ordered ranks through insertion and deletion", () => { - const index = new OrderedIndex((left, right) => left.key - right.key); - const values = Array.from({ length: 1000 }, (_, key) => ({ key })); - for (let offset = 0; offset < 10; offset += 1) { - for (let key = offset; key < values.length; key += 10) - index.add(values[key]); - } - expect(index.values()).toEqual(values); - for (const [position, value] of values.entries()) { - expect(index.at(position)).toBe(value); - expect(index.indexOf(value)).toBe(position); - } - for (const value of values.filter(({ key }) => key % 3 === 0)) - index.delete(value); - const remaining = values.filter(({ key }) => key % 3 !== 0); - expect(index.values()).toEqual(remaining); - for (const [position, value] of remaining.entries()) { - expect(index.indexOf(value)).toBe(position); - } - }); - test("indexes text styles by id runs and causal version", () => { - const styles = new TextStyleIndex(); - const outer = { - startId: { peer: 7n, counter: 0 }, - lamport: 1, - info: 0, - value: true, - }; - const inner = { - startId: { peer: 8n, counter: 0 }, - lamport: 2, - info: 0, - value: null, - }; - styles.add([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", outer); - styles.add([{ start: { peer: 1n, counter: 25 }, length: 50 }], "bold", inner); - expect(styles.metasAt({ peer: 1n, counter: 10 }).get("bold")).toBe(outer); - expect(styles.metasAt({ peer: 1n, counter: 50 }).get("bold")).toBe(inner); - expect(styles.metasAt({ peer: 1n, counter: 50 }, new Map([[7n, 1]])).get("bold")).toBe(outer); - expect(styles.rangeHasKey([{ start: { peer: 1n, counter: 30 }, length: 20 }], "bold")).toBe(false); - expect(styles.runsContainMeta([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", outer.startId)).toBe(true); - expect(styles.runsContainMeta([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", inner.startId)).toBe(false); - expect(styles.transitions([{ start: { peer: 1n, counter: 0 }, length: 100 }], "bold", new Map([[7n, 1]]), undefined)).toEqual([ - { - run: { start: { peer: 1n, counter: 25 }, length: 50 }, - before: outer, - after: inner, - }, - ]); - }); - test("indexes forward and reverse deletion runs in both directions", () => { - const deletions = new SequenceDeletionIndex(); - deletions.add({ peer: 1n, counter: 10 }, 5, { peer: 2n, counter: 100 }, 1); - deletions.add({ peer: 1n, counter: 12 }, 4, { peer: 3n, counter: 200 }, -1); - expect(deletions.deletionIdsAt({ peer: 1n, counter: 13 })).toEqual([ - { peer: 2n, counter: 103 }, - { peer: 3n, counter: 202 }, - ]); - expect(deletions.someDeletion({ peer: 1n, counter: 15 }, (id) => id.counter < 201)).toBe(true); - expect(deletions.everyDeletion({ peer: 1n, counter: 13 }, (id) => id.counter >= 100)).toBe(true); - expect(deletions.targetRunsDeletedBy(2n, 101, 104)).toEqual([ - { start: { peer: 1n, counter: 11 }, length: 3 }, - ]); - expect(deletions.targetRunsDeletedBy(3n, 201, 204)).toEqual([ - { start: { peer: 1n, counter: 12 }, length: 3 }, - ]); - }); - test("indexes visibility at an earlier causal version", () => { - const sequence = new SequenceIndex(); - const elements = [ - { id: { peer: 1n, counter: 0 }, deleted: false, deletedBy: [], value: "a" }, - { id: { peer: 2n, counter: 0 }, deleted: false, deletedBy: [], value: "b" }, - { id: { peer: 1n, counter: 1 }, deleted: false, deletedBy: [], value: "c" }, - ]; - sequence.insertAtPhysical(0, elements); - sequence.setDeleted(elements[0], true); - sequence.addDeletion(elements[0], { peer: 2n, counter: 1 }); - const beforeConcurrentPeer = sequence.causalView(new Map([ - [1n, 2], - [2n, 0], - ])); - expect(beforeConcurrentPeer.length).toBe(2); - expect(beforeConcurrentPeer.range(0, 2).map(({ value }) => value)).toEqual([ - "a", - "c", - ]); - expect(beforeConcurrentPeer.at(1)?.value).toBe("c"); - const mutableVersion = new Map([ - [1n, 2], - [2n, 0], - ]); - const immutableView = sequence.causalView(mutableVersion); - mutableVersion.set(2n, 2); - expect(immutableView.range(0, immutableView.length).map(({ value }) => value)).toEqual(["a", "c"]); - expect(sequence.causalView(new Map([ - [1n, 2], - [2n, 0], - ]))).toBe(beforeConcurrentPeer); - const withoutPeerOneTail = sequence.causalView(new Map([ - [1n, 1], - [2n, 2], - ])); - expect(withoutPeerOneTail.range(0, withoutPeerOneTail.length).map(({ value }) => value)).toEqual(["b"]); - expect(sequence.causalView(new Map([ - [1n, 2], - [2n, 0], - ]))).toBe(beforeConcurrentPeer); - const latest = sequence.causalView(new Map([ - [1n, 2], - [2n, 2], - ])); - expect(latest.range(0, latest.length).map(({ value }) => value)).toEqual(["b", "c"]); - sequence.moveBefore(elements[2], elements[0]); - const afterMove = sequence.causalView(new Map([ - [1n, 2], - [2n, 0], - ])); - expect(afterMove).not.toBe(beforeConcurrentPeer); - expect(afterMove.range(0, afterMove.length).map(({ value }) => value)).toEqual([ - "c", - "a", - ]); - }); - test("packs batch inserts into bounded sequence spans", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 10000 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: "x", - })); - sequence.insertAtPhysical(0, elements); - expect(sequence.spanCount).toBeLessThanOrEqual(Math.ceil(elements.length / 32)); - for (let index = 0; index < elements.length; index += 97) { - sequence.setDeleted(elements[index], true); - } - expect(sequence.spanCount).toBeLessThanOrEqual(Math.ceil(elements.length / 32)); - expect(sequence.visibleLength).toBe(elements.filter((element) => !element.deleted).length); - }); - test("skips deleted subtrees when finding visible neighbors", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 10000 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: "x", - })); - sequence.insertAtPhysical(0, elements); - sequence.deleteIdSpan({ peer: 1n, counter: 1 }, elements.length - 2, { - peer: 2n, - counter: 0, - }); - expect(sequence.nextVisible(elements[0])).toBe(elements.at(-1)); - expect(sequence.previousVisible(elements.at(-1))).toBe(elements[0]); - expect(sequence.nextVisible(elements[elements.length >>> 1])).toBe(elements.at(-1)); - expect(sequence.previousVisible(elements[elements.length >>> 1])).toBe(elements[0]); - }); - test("materializes lazy span deletions before restoring or restructuring", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 10000 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtPhysical(0, elements); - sequence.deleteIdSpan({ peer: 1n, counter: 0 }, elements.length, { - peer: 2n, - counter: 0, - }); - sequence.setDeleted(elements[5000], false); - expect(sequence.visible()).toEqual([elements[5000]]); - expect(sequence.visibleIndexOf(elements[5000])).toBe(0); - expect(sequence.atPhysical(4999)?.deleted).toBe(true); - const inserted = { - id: { peer: 3n, counter: 0 }, - deleted: false, - deletedBy: [], - value: "inserted", - }; - sequence.insertAtPhysical(5000, [inserted]); - expect(sequence.visible()).toEqual([inserted, elements[5000]]); - expect(sequence.all().filter((element) => !element.deleted)).toEqual([ - inserted, - elements[5000], - ]); - }); - test("keeps rank and range queries correct after lazy range restoration", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 10000 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtPhysical(0, elements); - const all = [{ start: { peer: 1n, counter: 0 }, length: elements.length }]; - sequence.setIdRunsDeleted(all); - sequence.setIdRunsVisible([{ start: { peer: 1n, counter: 100 }, length: 9800 }]); - expect(sequence.visibleLength).toBe(9800); - expect(sequence.atVisible(0)).toBe(elements[100]); - expect(sequence.atVisible(5000)).toBe(elements[5100]); - expect(sequence.atVisible(9799)).toBe(elements[9899]); - expect(sequence.visibleIndexOf(elements[5100])).toBe(5000); - expect(sequence.metricOffsetAtVisibleIndex(5000, "utf16")).toBe(5000); - expect(sequence.visibleIndexAtMetricOffset(5000, "utf16")).toBe(5000); - expect(sequence.visibleIdRuns(123, 9123)).toEqual([ - { start: { peer: 1n, counter: 223 }, length: 9000 }, - ]); - expect(sequence.visibleRange(4999, 5002)).toEqual([ - elements[5099], - elements[5100], - elements[5101], - ]); - expect(sequence.previousVisible(elements[100])).toBeUndefined(); - expect(sequence.nextVisible(elements[9899])).toBeUndefined(); - }); - test("deletes id spans while preserving causal delete order", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 96 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtPhysical(0, elements); - sequence.deleteIdSpan({ peer: 1n, counter: 20 }, 50, { - peer: 2n, - counter: 100, - }); - expect(sequence.visibleLength).toBe(46); - expect(sequence.visible().map(({ id }) => id.counter)).toEqual([ - ...Array.from({ length: 20 }, (_, counter) => counter), - ...Array.from({ length: 26 }, (_, counter) => counter + 70), - ]); - expect(sequence.elementsDeletedBy(2n, 100, 150).map(({ id }) => id.counter)).toEqual(Array.from({ length: 50 }, (_, counter) => counter + 20)); - const halfway = sequence.causalView(new Map([ - [1n, 96], - [2n, 125], - ])); - expect(halfway.length).toBe(71); - expect(halfway.range(19, 22).map(({ id }) => id.counter)).toEqual([19, 45, 46]); - const reversed = new SequenceIndex(); - const reversedElements = Array.from({ length: 8 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - reversed.insertAtPhysical(0, reversedElements); - reversed.deleteIdSpan({ peer: 1n, counter: 2 }, -4, { - peer: 2n, - counter: 10, - }); - const partialReverse = reversed.causalView(new Map([ - [1n, 8], - [2n, 12], - ])); - expect(partialReverse.range(0, partialReverse.length).map(({ id }) => id.counter)).toEqual([0, 1, 2, 3, 6, 7]); - }); - test("checks scalar and range deletions only on the restored ID runs", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 96 }, (_, counter) => ({ - id: { peer: counter === 95 ? 4n : 1n, counter: counter === 95 ? 0 : counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtPhysical(0, elements); - sequence.deleteIdSpan({ peer: 1n, counter: 0 }, 95, { - peer: 2n, - counter: 0, - }); - sequence.deleteElement(elements[95], { peer: 3n, counter: 0 }); - const restored = [{ start: { peer: 1n, counter: 0 }, length: 95 }]; - expect(sequence.canShowIdRunsAt(restored, new Map([ - [1n, 95], - [2n, 0], - [3n, 1], - [4n, 1], - ]))).toBe(true); - sequence.addDeletion(elements[0], { peer: 3n, counter: 1 }); - expect(sequence.canShowIdRunsAt(restored, new Map([ - [1n, 95], - [2n, 0], - [3n, 2], - [4n, 1], - ]))).toBe(false); - }); - test("coalesces adjacent single-element inserts into bounded spans", () => { - const sequence = new SequenceIndex(); - for (let counter = 0; counter < 10000; counter += 1) { - sequence.insertAtPhysical(sequence.allLength, [ - { - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - }, - ]); - } - expect(sequence.allLength).toBe(10000); - expect(sequence.spanCount).toBe(Math.ceil(10000 / 32)); - for (const counter of [0, 31, 32, 4999, 9999]) { - const element = sequence.findById({ peer: 1n, counter }); - expect(element?.value).toBe(String(counter)); - expect(sequence.physicalIndexOf(element)).toBe(counter); - } - }); - test("ranges across an excluded element", () => { - const sequence = new SequenceIndex(); - const base = Array.from({ length: 6 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String.fromCharCode(97 + counter), - })); - sequence.insertAtPhysical(0, base); - sequence.recordOperationId({ peer: 1n, counter: 6 }); - sequence.recordOperationId({ peer: 1n, counter: 7 }); - sequence.recordOperationId({ peer: 1n, counter: 8 }); - sequence.recordOperationId({ peer: 1n, counter: 9 }); - sequence.insertAtPhysical(3, [ - { - id: { peer: 1n, counter: 10 }, - deleted: false, - deletedBy: [], - value: "X", - }, - ]); - sequence.recordOperationId({ peer: 1n, counter: 11 }); - sequence.recordOperationId({ peer: 1n, counter: 12 }); - const view = sequence.causalView(new Map([[1n, 8]])); - expect(view.length).toBe(6); - expect(view.range(2, 4).map(({ value }) => value)).toEqual(["c", "d"]); - expect(view.idRuns(0, view.length)).toEqual([ - { start: { peer: 1n, counter: 0 }, length: 6 }, - ]); - }); - test("seeks sparse operation counters without walking numeric gaps", () => { - const sequence = new SequenceIndex(); - const first = { - id: { peer: 1n, counter: 0 }, - deleted: false, - deletedBy: [], - value: "a", - }; - const distant = { - id: { peer: 1n, counter: 1000000000 }, - deleted: false, - deletedBy: [], - value: "b", - }; - sequence.insertAtPhysical(0, [first, distant]); - sequence.setDeleted(first, true); - sequence.addDeletion(first, { peer: 2n, counter: 1000000000 }); - const early = sequence.causalView(new Map([ - [1n, 1], - [2n, 0], - ])); - expect(early.range(0, early.length).map(({ value }) => value)).toEqual(["a"]); - expect(sequence.elementsDeletedBy(2n, 999999999, 1000000001)).toEqual([first]); - }); - test("promotes a deletion counter to a set only for one-to-many deletes", () => { - const sequence = new SequenceIndex(); - const elements = Array.from({ length: 3 }, (_, counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtPhysical(0, elements); - for (const element of elements.slice(0, 2)) { - sequence.setDeleted(element, true); - sequence.addDeletion(element, { peer: 2n, counter: 0 }); - } - sequence.addDeletion(elements[0], { peer: 3n, counter: 0 }); - expect(sequence.elementsDeletedBy(2n, 0, 1)).toEqual(elements.slice(0, 2)); - const beforeDelete = sequence.causalView(new Map([ - [1n, 3], - [2n, 0], - [3n, 0], - ])); - expect(beforeDelete.range(0, beforeDelete.length)).toEqual(elements); - const afterSecondDelete = sequence.causalView(new Map([ - [1n, 3], - [2n, 0], - [3n, 1], - ])); - expect(afterSecondDelete.range(0, afterSecondDelete.length)).toEqual(elements.slice(1)); - }); - test("merges out-of-order counter runs", () => { - const sequence = new SequenceIndex(); - const counters = [10, 0, 2, 1, 1000000000, 11]; - const elements = counters.map((counter) => ({ - id: { peer: 1n, counter }, - deleted: false, - deletedBy: [], - value: String(counter), - })); - sequence.insertAtPhysical(0, elements); - const early = sequence.causalView(new Map([[1n, 2]])); - expect(early.range(0, early.length).map(({ id }) => id.counter)).toEqual([0, 1]); - for (let index = 0; index < elements.length; index += 1) { - sequence.setDeleted(elements[index], true); - sequence.addDeletion(elements[index], { - peer: 2n, - counter: counters[index], - }); - } - expect(sequence.elementsDeletedBy(2n, 1, 11).map(({ id }) => id.counter)).toEqual([ - 10, 2, 1, - ]); - }); -}); -function compressIdRuns(elements) { - const runs = []; - for (const { id } of elements) { - const previous = runs.at(-1); - if (previous !== undefined && - previous.start.peer === id.peer && - previous.start.counter + previous.length === id.counter) { - previous.length += 1; - } - else { - runs.push({ start: { ...id }, length: 1 }); - } - } - return runs; -} diff --git a/loro-js/tests/lz4.test.js b/loro-js/tests/lz4.test.js deleted file mode 100644 index a5f60350d..000000000 --- a/loro-js/tests/lz4.test.js +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { bytesToHex, decodeLz4Frame, encodeLz4FrameRaw, hexToBytes, } from "../src/codec/index"; -describe("LZ4 frame", () => { - test("decodes a compressed overlapping match", () => { - const frame = hexToBytes("04224d18604082040000001061010000000000"); - expect(new TextDecoder().decode(decodeLz4Frame(frame))).toBe("aaaaa"); - }); - test.each([0, 1, 65536, 65537, 300000])("round trips %i canonical frame bytes", (length) => { - const input = new Uint8Array(length); - for (let index = 0; index < input.length; index += 1) { - input[index] = index & 0xff; - } - expect(decodeLz4Frame(encodeLz4FrameRaw(input), { - requireCanonicalProfile: true, - })).toEqual(input); - }); - test("encodes an overlapping match with the required trailing literals", () => { - const input = new TextEncoder().encode("aaaaaaaaaaaaaaaaaa"); - const encoded = encodeLz4FrameRaw(input); - expect(bytesToHex(encoded)).toBe("04224d186040820a0000001861010050616161616100000000"); - expect(decodeLz4Frame(encoded)).toEqual(input); - }); - test("compresses highly repetitive blocks", () => { - const input = new Uint8Array(256 * 1024).fill(0x5a); - const encoded = encodeLz4FrameRaw(input); - expect(readFirstBlockInfo(encoded) & 2147483648).toBe(0); - expect(encoded.length).toBeLessThan(input.length / 100); - expect(decodeLz4Frame(encoded)).toEqual(input); - }); - test("stores incompressible blocks without expansion", () => { - const input = deterministicNoise(32 * 1024); - const encoded = encodeLz4FrameRaw(input); - expect(readFirstBlockInfo(encoded) & 2147483648).not.toBe(0); - expect(encoded.length).toBe(input.length + 15); - expect(decodeLz4Frame(encoded)).toEqual(input); - }); - test("round trips mixed noisy and repeated ranges", () => { - for (const [caseIndex, length] of [ - 13, 14, 15, 31, 255, 256, 4095, 65535, 65536, 65537, - ].entries()) { - const input = deterministicNoise(length, 305419896 + caseIndex); - for (let start = 97; start < input.length; start += 211) { - const copyLength = Math.min(80, input.length - start); - for (let index = 0; index < copyLength; index += 1) { - input[start + index] = input[index % 37]; - } - } - expect(decodeLz4Frame(encodeLz4FrameRaw(input))).toEqual(input); - } - }); -}); -function readFirstBlockInfo(frame) { - return new DataView(frame.buffer, frame.byteOffset + 7, 4).getUint32(0, true); -} -function deterministicNoise(length, seed = 305419896) { - const output = new Uint8Array(length); - let state = seed; - for (let index = 0; index < output.length; index += 1) { - state ^= state << 13; - state ^= state >>> 17; - state ^= state << 5; - output[index] = state >>> 24; - } - return output; -} diff --git a/loro-js/tests/primitives.test.js b/loro-js/tests/primitives.test.js deleted file mode 100644 index 877e239a7..000000000 --- a/loro-js/tests/primitives.test.js +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { ByteReader, ByteWriter, LORO_XXHASH_SEED, PostcardReader, PostcardWriter, bytesToHex, readSleb128, readUleb128, writeSleb128, writeUleb128, xxhash32, } from "../src/codec/index"; -describe("xxHash32", () => { - test.each([ - [new Uint8Array(), 0, 46947589], - [new TextEncoder().encode("a"), 0, 1426945110], - [new TextEncoder().encode("abc"), 0, 852579327], - [new TextEncoder().encode("message digest"), 0, 2090108052], - [new TextEncoder().encode("abc"), LORO_XXHASH_SEED, 2784525984], - ])("matches the reference vector", (input, seed, expected) => { - expect(xxhash32(input, seed)).toBe(expected); - }); -}); -describe("LEB128", () => { - test.each([0n, 1n, 127n, 128n, 0xffffffffn, 0xffffffffffffffffn])("round trips unsigned %s", (value) => { - const writer = new ByteWriter(); - writeUleb128(writer, value); - const reader = new ByteReader(writer.toUint8Array()); - expect(readUleb128(reader)).toBe(value); - expect(reader.remaining).toBe(0); - }); - test.each([-0x8000000000000000n, -65n, -1n, 0n, 63n, 64n, 0x7fffffffffffffffn])("round trips signed %s", (value) => { - const writer = new ByteWriter(); - writeSleb128(writer, value); - const reader = new ByteReader(writer.toUint8Array()); - expect(readSleb128(reader)).toBe(value); - expect(reader.remaining).toBe(0); - }); - test("uses signed LEB128 rather than postcard zigzag", () => { - const writer = new ByteWriter(); - writeSleb128(writer, -1n); - expect(bytesToHex(writer.toUint8Array())).toBe("7f"); - const postcard = new PostcardWriter(); - postcard.writeI32(-1); - expect(bytesToHex(postcard.toUint8Array())).toBe("01"); - }); -}); -describe("postcard primitives", () => { - test("round trips signed integers, strings, bytes and arrays", () => { - const writer = new PostcardWriter(); - writer.writeI32(-123); - writer.writeI64(-9007199254740993n); - writer.writeString("Loro 😀"); - writer.writeBytes(Uint8Array.of(0, 1, 255)); - writer.writeArray([1, 2, 300], (output, value) => output.writeU32(value)); - const reader = new PostcardReader(writer.toUint8Array()); - expect(reader.readI32()).toBe(-123); - expect(reader.readI64()).toBe(-9007199254740993n); - expect(reader.readString()).toBe("Loro 😀"); - expect(reader.readBytes()).toEqual(Uint8Array.of(0, 1, 255)); - expect(reader.readArray((input) => input.readU32())).toEqual([1, 2, 300]); - reader.assertEnd(); - }); -}); diff --git a/loro-js/tests/runtime.test.js b/loro-js/tests/runtime.test.js deleted file mode 100644 index de7517542..000000000 --- a/loro-js/tests/runtime.test.js +++ /dev/null @@ -1,1708 +0,0 @@ -import { readFileSync } from "node:fs"; -import { describe, expect, test } from "vitest"; -import { ContainerType, decodeFastSnapshot, decodePostcardVersionVector, decodeStateSnapshotStore, decodeSstable, encodeFastSnapshot, encodePostcardVersionVector, encodeStateSnapshotStore, encodeSstable, } from "../src/codec"; -import { Awareness, AwarenessWasm, Cursor, decodeFrontiers, decodeImportBlobMeta, encodeFrontiers, EphemeralStore, EphemeralStoreWasm, LORO_VERSION, LoroCounter, LoroDoc, LoroList, LoroMap, LoroMovableList, LoroText, redactJsonUpdates, UndoManager, VersionVector, } from "../src/index"; -const fixture = (name) => new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); -const withDuplicateTextId = (snapshot) => { - const stateStore = decodeStateSnapshotStore(snapshot.state); - if (stateStore.kind !== "sstable") - throw new Error("expected snapshot state"); - let changed = false; - const containers = stateStore.containers.map((entry) => { - const state = entry.wrapper.state; - if (changed || state.kind !== ContainerType.Text) - return entry; - const span = state.spans.find(({ length }) => length >= 2); - if (span === undefined) - return entry; - const peer = state.peers[Number(span.peerIndex)]; - const aliasPeerIndex = BigInt(state.peers.length); - changed = true; - return { - ...entry, - wrapper: { - ...entry.wrapper, - state: { - ...state, - peers: [...state.peers, peer], - spans: state.spans.flatMap((item) => item === span - ? [ - { ...span, length: 1 }, - { ...span, peerIndex: aliasPeerIndex, length: 1 }, - { ...span, counter: span.counter + 2, length: span.length - 2 }, - ].filter(({ length }) => length > 0) - : [item]), - }, - }, - }; - }); - if (!changed) - throw new Error("expected a splittable text span"); - return encodeFastSnapshot({ - ...snapshot, - state: encodeStateSnapshotStore({ ...stateStore, containers }, { compression: "none" }), - }); -}; -describe("loro-wasm-compatible runtime", () => { - test("keeps existing container handlers usable after freeing the doc wrapper", () => { - const doc = new LoroDoc(); - const text = doc.getText("text"); - const list = doc.getList("list"); - const map = doc.getMap("map"); - doc.free(); - text.insert(0, "123"); - list.insert(0, 1); - map.set("key", 8); - expect(text.toString()).toBe("123"); - expect(list.toJSON()).toEqual([1]); - expect(map.toJSON()).toEqual({ key: 8 }); - }); - test("edits map, list, text, movable-list, tree and counter containers", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - doc.getMap("map").set("answer", 42); - const list = doc.getList("list"); - list.push("a"); - list.push("b"); - list.delete(0, 1); - const text = doc.getText("text"); - text.insert(0, "a😀b"); - text.delete(0, 1); - const movable = doc.getMovableList("movable"); - movable.push(1); - movable.push(2); - movable.move(0, 1); - movable.set(0, 3); - const tree = doc.getTree("tree"); - const root = tree.createNode(); - root.data.set("title", "root"); - root.createNode(); - doc.getCounter("counter").increment(2.5); - expect(doc.toJSON()).toEqual({ - map: { answer: 42 }, - list: ["b"], - text: "😀b", - movable: [3, 1], - tree: [ - expect.objectContaining({ - id: root.id, - meta: { title: "root" }, - children: [expect.objectContaining({ parent: root.id })], - }), - ], - counter: 2.5, - }); - }); - test("returns wasm-shaped shallow values and deep values with container ids", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const map = doc.getMap("map"); - map.set("plain", 1); - const child = map.setContainer("body", new LoroText()); - child.insert(0, "hello"); - const tree = doc.getTree("tree"); - const node = tree.createNode(); - node.data.set("name", "root"); - expect(map.getShallowValue()).toEqual({ plain: 1, body: child.id }); - expect(doc.getShallowValue()).toEqual({ map: map.id, tree: tree.id }); - expect(tree.getShallowValue()).toEqual([ - { - id: node.id, - parent: null, - index: 0, - fractional_index: "80", - meta: node.data.id, - children: [], - }, - ]); - expect(tree.toJSON()).toEqual([ - { - id: node.id, - parent: null, - index: 0, - fractional_index: "80", - meta: { name: "root" }, - children: [], - }, - ]); - expect(doc.getDeepValueWithID()).toMatchObject({ - map: { - cid: map.id, - value: { - body: { cid: child.id, value: "hello" }, - plain: 1, - }, - }, - }); - }); - test("evaluates JSONPath selectors, recursion, slices, and filters", () => { - const doc = new LoroDoc(); - const store = doc.getMap("store"); - store.set("books", [ - { title: "1984", author: "George Orwell", price: 10, available: true }, - { title: "Animal Farm", author: "George Orwell", price: 8, available: true }, - { title: "Brave New World", author: "Aldous Huxley", price: 12, available: false }, - { title: "Fahrenheit 451", author: "Ray Bradbury", price: 9, available: true }, - { title: "Pride and Prejudice", author: "Jane Austen", price: 7, available: true }, - ]); - store.set("featured_author", "George Orwell"); - store.set("featured_authors", ["George Orwell", "Jane Austen"]); - const body = store.setContainer("body", new LoroText()); - body.insert(0, "content"); - const outline = doc.getTree("outline"); - const firstRoot = outline.createNode(); - firstRoot.data.set("label", "first"); - firstRoot.createNode().data.set("label", "child"); - outline.createNode().data.set("label", "last"); - expect(doc.JSONPath("$['store'].books[0].title")).toEqual(["1984"]); - expect(doc.JSONPath("$.store.books[*].title")).toEqual([ - "1984", - "Animal Farm", - "Brave New World", - "Fahrenheit 451", - "Pride and Prejudice", - ]); - expect(doc.JSONPath("$..title")).toHaveLength(5); - expect(doc.JSONPath("$.store.books[0,2,-1].title")).toEqual([ - "1984", - "Brave New World", - "Pride and Prejudice", - ]); - expect(doc.JSONPath("$.store.books[0:5:2].title")).toEqual([ - "1984", - "Brave New World", - "Pride and Prejudice", - ]); - expect(doc.JSONPath("$.store.books[?(@.author == $.store.featured_author && @.price < 10)].title")).toEqual(["Animal Farm"]); - expect(doc.JSONPath("$.store.books[?(@.author in $.store.featured_authors)].title")).toEqual(["1984", "Animal Farm", "Pride and Prejudice"]); - expect(doc.JSONPath("$..[?(@.title contains 'Farm')].title")).toEqual([ - "Animal Farm", - ]); - expect(doc.JSONPath("$.store.books[?(count(@.title) == 1)].title")).toHaveLength(5); - expect(doc.JSONPath("$.store.body")).toEqual([body]); - expect(doc.JSONPath("$.outline[0].meta.label")).toEqual(["first"]); - expect(doc.JSONPath("$.outline[-1].meta.label")).toEqual(["last"]); - expect(doc.JSONPath("$")).toEqual([{ store, outline }]); - expect(doc.getByPath("outline/0/0/label")).toBe("child"); - doc.commit(); - let subscriptionHits = 0; - const unsubscribe = doc.subscribeJsonpath("$.store.featured_author", () => { - subscriptionHits += 1; - }); - firstRoot.data.set("unrelated", true); - doc.commit(); - expect(subscriptionHits).toBe(0); - store.set("featured_author", "Jane Austen"); - doc.commit(); - expect(subscriptionHits).toBe(1); - unsubscribe(); - expect(() => doc.subscribeJsonpath("store", () => { })).toThrow(/start with/u); - }); - test("syncs updates in both directions", () => { - const left = new LoroDoc(); - left.setPeerId(1); - left.getList("list").push("A"); - left.getList("list").push("B"); - left.getText("text").insert(0, "hello"); - const right = new LoroDoc(); - right.setPeerId(2); - right.import(left.export({ mode: "update" })); - expect(right.toJSON()).toEqual(left.toJSON()); - const from = right.oplogVersion(); - right.getMap("map").set("x", true); - right.getList("list").delete(0, 1); - left.import(right.export({ mode: "update", from })); - expect(left.toJSON()).toEqual(right.toJSON()); - }); - test("slices update exports at operation boundaries", () => { - const source = new LoroDoc(); - source.setPeerId(1); - source.getText("text").insert(0, "abcde"); - source.commit(); - const prefix = source.export({ - mode: "updates-in-range", - spans: [{ id: { peer: "1", counter: 0 }, len: 2 }], - }); - const version = new VersionVector(new Map([["1", 2]])); - const suffix = source.export({ mode: "update", from: version }); - const target = new LoroDoc(); - target.import(prefix); - expect(target.getText("text").toString()).toBe("ab"); - target.import(suffix); - expect(target.getText("text").toString()).toBe("abcde"); - }); - test("converges when updates are imported in opposite orders", () => { - const base = new LoroDoc(); - base.setPeerId(1); - base.getList("list").push("base"); - base.getText("text").insert(0, "x"); - const initial = base.export({ mode: "update" }); - const left = new LoroDoc(); - left.setPeerId(2); - left.import(initial); - const right = new LoroDoc(); - right.setPeerId(3); - right.import(initial); - const version = left.oplogVersion(); - left.getList("list").insert(0, "left"); - left.getText("text").insert(0, "L"); - left.getMap("map").set("winner", "left"); - const leftUpdate = left.export({ mode: "update", from: version }); - right.getList("list").insert(0, "right"); - right.getText("text").insert(0, "R"); - right.getMap("map").set("winner", "right"); - const rightUpdate = right.export({ mode: "update", from: version }); - left.import(rightUpdate); - right.import(leftUpdate); - expect(left.toJSON()).toEqual(right.toJSON()); - }); - test("keeps concurrent backward insertions in Fugue order", () => { - const left = new LoroDoc(); - left.setPeerId(1); - for (const character of ["o", "l", "l", "e", "H"]) { - left.getText("text").insert(0, character); - } - const right = new LoroDoc(); - right.setPeerId(2); - for (const character of ["!", "d", "l", "r", "o", "W", " "]) { - right.getText("text").insert(0, character); - } - const leftUpdate = left.export({ mode: "update" }); - const rightUpdate = right.export({ mode: "update" }); - left.import(rightUpdate); - right.import(leftUpdate); - expect(left.getText("text").toString()).toBe("Hello World!"); - expect(right.getText("text").toString()).toBe("Hello World!"); - }); - test("does not interleave a concurrent insertion into an existing Fugue run", () => { - const first = new LoroDoc(); - first.setPeerId(1); - const concurrent = new LoroDoc(); - concurrent.setPeerId(2); - const seed = new LoroDoc(); - seed.setPeerId(3); - seed.getText("text").insert(0, "2"); - first.import(seed.export({ mode: "update" })); - first.getText("text").insert(0, "1"); - concurrent.getText("text").insert(0, "b"); - first.import(concurrent.export({ mode: "update" })); - expect(first.getText("text").toString()).toBe("b12"); - }); - test("checks out partial changes and attaches back to the latest state", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "abc"); - doc.commit(); - doc.checkout([{ peer: "1", counter: 1 }]); - expect(text.toString()).toBe("ab"); - expect(doc.isDetached()).toBe(true); - expect(doc.version().toJSON()).toEqual(new Map([["1", 2]])); - expect(doc.oplogVersion().toJSON()).toEqual(new Map([["1", 3]])); - expect(() => text.push("x")).toThrow(/detached document/u); - doc.attach(); - expect(text.toString()).toBe("abc"); - expect(doc.isDetached()).toBe(false); - }); - test("records imports in the oplog while the state is detached", () => { - const source = new LoroDoc(); - source.setPeerId(1); - source.getText("text").insert(0, "base"); - const base = source.export({ mode: "update" }); - const doc = new LoroDoc(); - doc.setPeerId(2); - doc.import(base); - const checkedOut = doc.frontiers(); - doc.checkout(checkedOut); - source.getText("text").push(" latest"); - doc.import(source.export({ mode: "update", from: doc.oplogVersion() })); - expect(doc.getText("text").toString()).toBe("base"); - expect(doc.oplogVersion().compare(doc.version())).toBe(1); - doc.checkoutToLatest(); - expect(doc.getText("text").toString()).toBe("base latest"); - }); - test("holds causally incomplete updates pending until dependencies arrive", () => { - const source = new LoroDoc(); - source.setPeerId(1); - source.getList("list").push("first"); - source.commit(); - const firstVersion = source.oplogVersion(); - const firstUpdate = source.export({ mode: "update" }); - source.getList("list").push("second"); - source.commit(); - const secondUpdate = source.export({ mode: "update", from: firstVersion }); - const target = new LoroDoc(); - const pending = target.import(secondUpdate); - expect(target.getList("list").toArray()).toEqual([]); - expect(pending.success.size).toBe(0); - expect(pending.pending).toEqual(new Map([["1", { start: 1, end: 2 }]])); - const applied = target.import(firstUpdate); - expect(target.getList("list").toArray()).toEqual(["first", "second"]); - expect(applied.success).toEqual(new Map([["1", { start: 0, end: 2 }]])); - expect(applied.pending).toBeNull(); - }); - test("imports the unseen suffix of a change that overlaps local history", () => { - const source = new LoroDoc(); - source.setPeerId(1); - source.getText("text").insert(0, "a"); - source.commit(); - const prefix = source.export({ mode: "update" }); - source.getText("text").insert(1, "b"); - source.commit(); - expect(source.changeCount()).toBe(1); - const target = new LoroDoc(); - target.import(prefix); - const status = target.import(source.export({ mode: "snapshot" })); - expect(target.getText("text").toString()).toBe("ab"); - expect(status.success).toEqual(new Map([["1", { start: 1, end: 2 }]])); - expect(status.pending).toBeNull(); - }); - test("forkAt keeps only history reachable from the requested frontiers", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - doc.getList("list").push("first"); - doc.commit(); - const first = doc.frontiers(); - doc.getList("list").push("second"); - doc.commit(); - const fork = doc.forkAt(first); - expect(fork.getList("list").toArray()).toEqual(["first"]); - expect(fork.opCount()).toBe(1); - }); - test("queries version spans, ancestors and changed containers", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - doc.getText("text").insert(0, "Hello"); - doc.commit({ message: "first" }); - const first = doc.frontiers(); - doc.getText("text").push(" World"); - doc.getMap("map").set("ready", true); - doc.commit({ message: "second" }); - const second = doc.frontiers(); - expect(doc.findIdSpansBetween(first, second)).toEqual({ - retreat: [], - forward: [{ peer: "1", counter: 5, length: 7 }], - }); - expect(doc.findIdSpansBetween(second, first)).toEqual({ - retreat: [{ peer: "1", counter: 5, length: 7 }], - forward: [], - }); - expect(new Set(doc.getChangedContainersIn({ peer: "1", counter: 5 }, 7))).toEqual(new Set([doc.getText("text").id, doc.getMap("map").id])); - const messages = []; - doc.travelChangeAncestors(second, (change) => { - messages.push(change.message); - }); - expect(messages).toEqual(["first", "second"]); - }); - test("emits compatible text, list and map diffs", () => { - const doc = new LoroDoc(); - let batch; - doc.subscribe((event) => (batch = event)); - const text = doc.getText("text"); - text.insert(0, "3"); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ type: "text", diff: [{ insert: "3" }] }); - text.insert(1, "12"); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ - type: "text", - diff: [{ retain: 1 }, { insert: "12" }], - }); - const list = doc.getList("list"); - list.insert(0, "3"); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ type: "list", diff: [{ insert: ["3"] }] }); - list.insert(1, "12"); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ - type: "list", - diff: [{ retain: 1 }, { insert: ["12"] }], - }); - const map = doc.getMap("map"); - map.set("a", 1); - map.set("b", 2); - doc.commit(); - expect(batch?.events[0]?.diff).toEqual({ - type: "map", - updated: { a: 1, b: 2 }, - }); - }); - test("emits the final map value after a delete and restore in one commit", () => { - const doc = new LoroDoc(); - doc.setPeerId(2); - const map = doc.getMap("map"); - map.set("title", {}); - doc.commit(); - const batches = []; - doc.subscribe((event) => batches.push(event)); - map.delete("title"); - map.set("title", {}); - doc.attach(); - expect(batches).toHaveLength(1); - expect(batches[0]?.events).toEqual([ - { - target: map.id, - diff: { type: "map", updated: { title: {} } }, - path: ["map"], - }, - ]); - }); - test("preserves visible map transitions but suppresses same-value import winners", () => { - const source = new LoroDoc(); - source.setPeerId(2); - const sourceMap = source.getMap("map"); - sourceMap.set("title", {}); - source.commit(); - const target = new LoroDoc(); - target.setPeerId(1); - target.import(source.export({ mode: "update" })); - const batches = []; - target.subscribe((event) => batches.push(event)); - sourceMap.delete("title"); - sourceMap.set("title", {}); - source.commit(); - target.import(source.export({ mode: "update" })); - expect(batches.map((batch) => batch.events[0]?.diff)).toEqual([ - { type: "map", updated: { title: {} } }, - ]); - const concurrent = new LoroDoc(); - concurrent.setPeerId(3); - concurrent.getMap("map").set("title", {}); - concurrent.commit(); - batches.length = 0; - target.import(concurrent.export({ mode: "update" })); - expect(batches).toEqual([]); - }); - test("does not emit changes for tree metadata deleted in the same commit", () => { - const doc = new LoroDoc(); - const tree = doc.getTree("tree"); - const batches = []; - doc.subscribe((event) => batches.push(event)); - const node = tree.createNode(); - node.data.set("value", []); - tree.delete(node.id); - doc.commit(); - expect(batches).toEqual([]); - }); - test("calculates raw and JSON-compatible diffs without moving document state", () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - const child = map.setContainer("child", new LoroMap()); - child.set("value", 1); - doc.getList("list").push("item"); - doc.commit(); - const latest = doc.frontiers(); - const raw = doc.diff([], latest, false); - const rawMap = raw.find(([id]) => id === map.id)?.[1]; - expect(rawMap).toEqual({ type: "map", updated: { child } }); - const json = doc.diff([], latest); - const jsonMap = json.find(([id]) => id === map.id)?.[1]; - expect(jsonMap).toEqual({ - type: "map", - updated: { child: `🦜:${child.id}` }, - }); - expect(doc.toJSON()).toEqual({ map: { child: { value: 1 } }, list: ["item"] }); - }); - test("keeps map tombstones that are visible only in operation history", () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - map.set("created-then-deleted", 1); - doc.commit(); - map.delete("created-then-deleted"); - doc.commit(); - expect(doc.diff([], doc.frontiers(), false)).toEqual([ - [map.id, { type: "map", updated: { "created-then-deleted": undefined } }], - ]); - }); - test("applies raw and serialized diffs with fresh child and tree ids", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const map = source.getMap("map"); - const child = map.setContainer("child", new LoroText()); - child.insert(0, "Hello🙂"); - child.mark({ start: 0, end: 5 }, "bold", true); - map.ensureMergeableCounter("revision").increment(3); - const listChild = source.getList("list").pushContainer(new LoroMap()); - listChild.set("nested", true); - const treeNode = source.getTree("tree").createNode(); - treeNode.data.set("name", "root"); - source.commit(); - const raw = source.diff([], source.frontiers(), false); - const serialized = JSON.parse(JSON.stringify(source.diff([], source.frontiers()))); - for (const batch of [raw, serialized]) { - const target = new LoroDoc(); - target.setPeerId(2); - target.applyDiff(batch); - const targetChild = target.getMap("map").get("child"); - expect(targetChild.id).not.toBe(child.id); - expect(targetChild.toDelta()).toEqual(child.toDelta()); - expect(target.toJSON()).toMatchObject({ - map: { child: "Hello🙂", revision: 3 }, - list: [{ nested: true }], - }); - const targetRoot = target.getTree("tree").roots()[0]; - expect(targetRoot?.id).not.toBe(treeNode.id); - expect(targetRoot?.data.toJSON()).toEqual({ name: "root" }); - } - }); - test("reverts by generating operations, including recreated child containers", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const list = doc.getList("list"); - list.push("item1"); - list.push("item2"); - const child = list.pushContainer(new LoroText()); - child.insert(0, "Hello🙂"); - doc.getText("title").insert(0, "A🙂B"); - doc.commit(); - const populated = doc.frontiers(); - child.delete(0, child.length); - list.clear(); - doc.getText("title").update("changed"); - doc.commit(); - doc.revertTo(populated); - expect(doc.toJSON()).toMatchObject({ - list: ["item1", "item2", "Hello🙂"], - title: "A🙂B", - }); - expect(list.get(2).id).not.toBe(child.id); - doc.commit(); - doc.revertTo([]); - expect(doc.toJSON()).toEqual({ list: [], title: "" }); - expect(doc.isDetached()).toBe(false); - }); - test("undoes and redoes local commit groups", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const pushed = []; - const popped = []; - const undo = new UndoManager(doc, { - mergeInterval: 0, - onPush: (isUndo, range) => { - pushed.push([isUndo, range.start, range.end]); - return { value: range.end, cursors: [] }; - }, - onPop: (isUndo) => popped.push(isUndo), - }); - doc.getText("text").insert(0, "hello"); - doc.commit(); - doc.getText("text").insert(5, " world"); - doc.commit(); - expect(undo.canUndo()).toBe(true); - expect(undo.undo()).toBe(true); - expect(doc.toJSON()).toEqual({ text: "hello" }); - expect(undo.undo()).toBe(true); - expect(doc.toJSON()).toEqual({ text: "" }); - expect(undo.canUndo()).toBe(false); - expect(undo.redo()).toBe(true); - expect(doc.toJSON()).toEqual({ text: "hello" }); - expect(undo.redo()).toBe(true); - expect(doc.toJSON()).toEqual({ text: "hello world" }); - expect(undo.canRedo()).toBe(false); - expect(pushed).toHaveLength(6); - expect(popped).toEqual([true, true, false, false]); - }); - test("trims undo history from the front", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const undo = new UndoManager(doc, { mergeInterval: 0, maxUndoSteps: 2 }); - for (const value of ["a", "b", "c"]) { - text.push(value); - doc.commit(); - } - expect(undo.undo()).toBe(true); - expect(undo.undo()).toBe(true); - expect(undo.undo()).toBe(false); - expect(text.toString()).toBe("a"); - }); - test("undo preserves remote and excluded text edits", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const undo = new UndoManager(doc, { - mergeInterval: 0, - excludeOriginPrefixes: ["sys:"], - }); - text.insert(0, "hello"); - doc.commit(); - text.insert(0, "1"); - doc.commit({ origin: "sys:test" }); - text.insert(2, "2"); - doc.commit({ origin: "sys:test" }); - text.insert(4, "3"); - doc.commit({ origin: "sys:test" }); - expect(undo.undo()).toBe(true); - expect(text.toString()).toBe("123"); - expect(undo.redo()).toBe(true); - expect(text.toString()).toBe("1h2e3llo"); - const remote = new LoroDoc(); - remote.setPeerId(2); - remote.import(doc.export({ mode: "snapshot" })); - remote.getText("text").insert(0, "R"); - remote.commit(); - doc.import(remote.export({ mode: "update" })); - expect(undo.undo()).toBe(true); - expect(text.toString()).toBe("R123"); - }); - test("undo follows the local peer after detached checkout", () => { - const doc = new LoroDoc(); - const undo = new UndoManager(doc, { mergeInterval: 0 }); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "Hello"); - doc.commit(); - text.insert(5, " world!"); - doc.commit(); - doc.setDetachedEditing(true); - doc.checkout([{ peer: "1", counter: 4 }]); - expect(undo.canUndo()).toBe(false); - text.insert(5, " alice!"); - doc.commit(); - expect(undo.undo()).toBe(true); - expect(text.toString()).toBe("Hello"); - }); - test("round-trips compressed and uncompressed JSON updates", () => { - const source = new LoroDoc(); - source.setPeerId(19); - source.getText("text").insert(0, "Hello🙂"); - const map = source.getMap("map"); - const child = map.setContainer("child", new LoroMap()); - child.set("value", { nested: [1, true, null] }); - source.getList("list").push("item"); - const movable = source.getMovableList("movable"); - movable.push("first"); - movable.push("second"); - movable.move(1, 0); - movable.set(1, "edited"); - const node = source.getTree("tree").createNode(); - node.data.set("name", "root"); - source.getCounter("counter").increment(2.5); - source.commit(); - const compressed = source.exportJsonUpdates(); - expect(compressed.peers).toEqual(["19"]); - expect(compressed.changes[0]?.id).toBe("0@0"); - expect(source.getOpsInChange({ peer: "19", counter: 0 })).toEqual(source.exportJsonInIdSpan({ - peer: "19", - counter: 0, - length: source.opCount(), - })[0]?.ops); - const uncompressed = source.exportJsonUpdates(undefined, undefined, false); - expect(uncompressed.peers).toBeNull(); - expect(uncompressed.changes[0]?.id).toBe("0@19"); - for (const update of [compressed, JSON.stringify(compressed), uncompressed]) { - const target = new LoroDoc(); - target.importJsonUpdates(update); - expect(target.toJSON()).toEqual(source.toJSON()); - expect(target.export({ mode: "update" })).toBeInstanceOf(Uint8Array); - } - }); - test("redacts JSON update content while preserving child-container structure", () => { - const source = new LoroDoc(); - source.setPeerId(1); - source.getText("text").insert(0, "secret"); - source.getMap("map").set("token", "private"); - const child = source.getMap("map").setContainer("child", new LoroMap()); - child.set("inside", 42); - source.getList("list").push("hidden"); - source.commit(); - const redacted = redactJsonUpdates(source.exportJsonUpdates(), { - "1": [0, source.opCount()], - }); - const target = new LoroDoc(); - target.importJsonUpdates(redacted); - expect(target.toJSON()).toEqual({ - text: "������", - map: { token: null, child: { inside: null } }, - list: [null], - }); - }); - test("exports pending operations as JSON without committing", () => { - const doc = new LoroDoc(); - doc.setPeerId(7); - doc.getText("text").insert(0, "pending"); - const pending = doc.getUncommittedOpsAsJson(); - expect(pending).toMatchObject({ peers: null }); - expect(pending?.changes[0]).toMatchObject({ id: "0@7", ops: [{ counter: 0 }] }); - expect(doc.changeCount()).toBe(0); - const target = new LoroDoc(); - target.importJsonUpdates(pending); - expect(target.toJSON()).toEqual({ text: "pending" }); - }); - test("decodes update, snapshot, and shallow-snapshot blob metadata", () => { - const doc = new LoroDoc(); - doc.setPeerId(5); - doc.setChangeMergeInterval(9); - doc.getText("text").insert(0, "abc"); - doc.commit({ timestamp: 10 }); - const first = doc.version(); - const firstFrontiers = doc.frontiers(); - doc.getMap("map").set("value", 1); - doc.commit({ timestamp: 20 }); - const update = decodeImportBlobMeta(doc.export({ mode: "update", from: first })); - expect(update.mode).toBe("update"); - expect(update.partialStartVersionVector.get("5")).toBe(3); - expect(update.partialEndVersionVector.get("5")).toBe(4); - expect(update.startTimestamp).toBe(20); - expect(update.endTimestamp).toBe(20); - expect(update.changeNum).toBe(1); - const snapshot = decodeImportBlobMeta(doc.export({ mode: "snapshot" })); - expect(snapshot).toMatchObject({ - mode: "snapshot", - startTimestamp: 0, - endTimestamp: 20, - changeNum: 2, - }); - expect(snapshot.partialEndVersionVector.get("5")).toBe(4); - const shallow = decodeImportBlobMeta(doc.export({ mode: "shallow-snapshot", frontiers: firstFrontiers })); - expect(shallow.mode).toBe("shallow-snapshot"); - expect(shallow.startFrontiers).toEqual(firstFrontiers); - expect(shallow.partialStartVersionVector.get("5")).toBe(2); - expect(shallow.partialEndVersionVector.get("5")).toBe(4); - }); - test("merges continuous changes using the wasm change interval", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const updates = []; - source.subscribeLocalUpdates((update) => updates.push(update)); - source.getMap("map").set("first", { shared: 1, left: true }); - source.commit({ timestamp: 110 }); - source.getMap("map").set("second", { right: 2, shared: 3 }); - source.commit({ timestamp: 120 }); - expect(source.changeCount()).toBe(1); - expect(source.getAllChanges().get("1")).toEqual([ - expect.objectContaining({ counter: 0, length: 2, timestamp: 110 }), - ]); - expect(source.exportJsonUpdates().changes).toHaveLength(1); - expect(() => source.debugHistory()).not.toThrow(); - const target = new LoroDoc(); - for (const update of updates) - target.import(update); - expect(target.changeCount()).toBe(1); - expect(target.toJSON()).toEqual(source.toJSON()); - const snapshotTarget = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); - expect(snapshotTarget.toJSON()).toEqual({ - map: { - first: { shared: 1, left: true }, - second: { right: 2, shared: 3 }, - }, - }); - const unmerged = new LoroDoc(); - unmerged.setPeerId(1); - unmerged.setChangeMergeInterval(9); - unmerged.getText("text").insert(0, "a"); - unmerged.commit({ timestamp: 110 }); - unmerged.getText("text").insert(1, "b"); - unmerged.commit({ timestamp: 120 }); - expect(unmerged.getAllChanges().get("1")).toHaveLength(2); - }); - test("appends many merged commits without changing their public history", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const updates = []; - source.subscribeLocalUpdates((update) => updates.push(update)); - const text = source.getText("text"); - for (let index = 0; index < 512; index += 1) { - text.insert(index, "x"); - source.commit(); - } - expect(source.changeCount()).toBe(1); - expect(source.getAllChanges().get("1")).toEqual([ - expect.objectContaining({ counter: 0, length: 512 }), - ]); - const target = new LoroDoc(); - target.importBatch(updates); - expect(target.getText("text").toString()).toBe("x".repeat(512)); - expect(target.changeCount()).toBe(1); - }); - test("coalesces consecutive list inserts in one pending change", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const list = source.getList("list"); - for (let index = 0; index < 128; index += 1) - list.insert(index, index); - expect(source.getUncommittedOpsAsJson()?.changes[0]?.ops).toHaveLength(1); - source.commit(); - expect(source.exportJsonUpdates().changes[0]?.ops).toHaveLength(1); - const target = new LoroDoc(); - target.import(source.export({ mode: "update" })); - expect(target.getList("list").toArray()).toEqual(Array.from({ length: 128 }, (_, index) => index)); - }); - test("preserves next commit options across implicit empty commits only", () => { - const implicit = new LoroDoc(); - implicit.setPeerId(1); - implicit.setNextCommitMessage("kept"); - implicit.export({ mode: "update" }); - implicit.getText("text").insert(0, "a"); - implicit.commit(); - expect(implicit.getAllChanges().get("1")?.[0]?.message).toBe("kept"); - const explicit = new LoroDoc(); - explicit.setPeerId(2); - explicit.setNextCommitMessage("discarded"); - explicit.commit(); - explicit.getText("text").insert(0, "b"); - explicit.commit(); - expect(explicit.getAllChanges().get("2")?.[0]?.message).toBeUndefined(); - }); - test("groups all changes by peer like the wasm Map API", () => { - const first = new LoroDoc(); - first.setPeerId(1); - first.getText("text").insert(0, "a"); - first.commit(); - const second = new LoroDoc(); - second.setPeerId(2); - second.getText("text").insert(0, "b"); - second.commit(); - const target = new LoroDoc(); - target.import(first.export({ mode: "update" })); - target.import(second.export({ mode: "update" })); - expect([...target.getAllChanges().keys()].sort()).toEqual(["1", "2"]); - }); - test("gets the latest peer change at or before a lamport", () => { - const first = new LoroDoc(); - first.setPeerId(1); - const second = new LoroDoc(); - second.setPeerId(2); - first.getText("text").insert(0, "01234"); - second.import(first.export({ mode: "update" })); - second.getText("text").insert(0, "56789"); - first.import(second.export({ mode: "update" })); - first.getText("text").insert(0, "01234"); - first.commit(); - expect(first.getChangeAtLamport("1", 1)).toMatchObject({ - peer: "1", - lamport: 0, - length: 5, - }); - expect(first.getChangeAtLamport("1", 7)?.lamport).toBe(0); - expect(first.getChangeAtLamport("1", 10)?.lamport).toBe(10); - expect(first.getChangeAtLamport("1", 20)?.lamport).toBe(10); - expect(first.getChangeAtLamport("111", 20)).toBeUndefined(); - expect(() => first.getChangeAt({ peer: "1", counter: 99 })).toThrow(/unknown/u); - expect(() => first.getOpsInChange({ peer: "1", counter: 99 })).toThrow(/unknown/u); - }); - test("emits absolute paths and bubbles descendant events to container subscribers", () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - const seen = []; - map.subscribe((event) => seen.push(event)); - const subMap = map.setContainer("sub", new LoroMap()); - subMap.set("value", 1); - doc.commit(); - expect(seen.at(-1)?.events.find((event) => event.target === subMap.id)?.path).toEqual(["map", "sub"]); - const list = subMap.setContainer("list", new LoroList()); - list.insert(0, "item"); - const text = list.insertContainer(1, new LoroText()); - doc.commit(); - text.insert(0, "nested"); - doc.commit(); - const textEvent = seen.at(-1)?.events.find((event) => event.target === text.id); - expect(textEvent?.path).toEqual(["map", "sub", "list", 1]); - expect(seen.at(-1)?.currentTarget).toBe(map.id); - expect(doc.getPathToContainer(text.id)).toEqual(["map", "sub", "list", 1]); - text._parentLink = { container: list }; - expect(doc.getPathToContainer(text.id)).toEqual(["map", "sub", "list", 1]); - expect(text._parentLink.binding?.kind).toBe("sequence"); - expect(doc.getByPath("map/sub")).toBe(subMap); - expect(doc.getByPath("map/sub/list/1")).toBe(text); - }); - test("merges first-peer callbacks into the commit and supports ChangeModifier", () => { - const doc = new LoroDoc(); - doc.setPeerId(9); - doc.subscribeFirstCommitFromPeer(({ peer }) => { - doc.getMap("peers").set(peer, `user-${peer}`); - }); - doc.subscribePreCommit(({ changeMeta, modifier }) => { - expect(changeMeta.length).toBe(2); - modifier.setMessage("signed").setTimestamp(123); - }); - doc.getList("list").push("value"); - doc.commit(); - expect(doc.changeCount()).toBe(1); - expect(doc.opCount()).toBe(2); - expect(doc.getMap("peers").get("9")).toBe("user-9"); - expect(doc.getAllChanges().get("9")?.[0]).toMatchObject({ - message: "signed", - timestamp: 123, - }); - }); - test("round trips a TypeScript snapshot", () => { - const source = new LoroDoc(); - source.setPeerId(7); - source.getMap("map").set("nested", { a: [1, true, "x"] }); - source.getText("text").insert(0, "snapshot"); - source.getCounter("counter").increment(3); - source.commit({ message: "snapshot" }); - const target = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); - expect(target.toJSON()).toEqual(source.toJSON()); - expect(target.getAllChanges()).toEqual(source.getAllChanges()); - }); - test("coalesces consecutive text IDs without splitting Unicode scalars", () => { - const source = new LoroDoc(); - source.setPeerId(7); - const text = source.getText("text"); - const value = `${"a".repeat(31)}😀b`; - text.insert(0, `${"a".repeat(31)}😀`); - text.push("b"); - source.commit(); - const snapshot = decodeFastSnapshot(source.export({ mode: "snapshot" })); - const stateStore = decodeStateSnapshotStore(snapshot.state); - expect(stateStore.kind).toBe("sstable"); - if (stateStore.kind !== "sstable") - throw new Error("expected snapshot state"); - const textState = stateStore.containers.find(({ wrapper }) => wrapper.state.kind === ContainerType.Text)?.wrapper.state; - expect(textState?.kind).toBe(ContainerType.Text); - if (textState?.kind !== ContainerType.Text) - throw new Error("expected text state"); - expect(textState.text).toBe(value); - expect(textState.spans.filter(({ length }) => length > 0)).toEqual([ - { peerIndex: 0n, counter: 0, lamportSub: 0, length: 33 }, - ]); - const target = LoroDoc.fromSnapshot(encodeFastSnapshot(snapshot)); - expect(target.getText("text").toString()).toBe(value); - }); - test("bulk-loads multi-span text snapshots and remains editable", () => { - const source = new LoroDoc(); - source.setPeerId(7); - const sourceText = source.getText("text"); - sourceText.insert(0, Array.from({ length: 80 }, (_, index) => `${index % 10}😀`).join("")); - sourceText.delete(6, 6); - sourceText.delete(60, 9); - source.commit(); - const target = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); - const targetText = target.getText("text"); - expect(targetText.toString()).toBe(sourceText.toString()); - targetText.insert(0, "<"); - targetText.insert(31, "中😀"); - targetText.push(">"); - targetText.delete(1, 3); - target.commit(); - const restored = LoroDoc.fromSnapshot(target.export({ mode: "snapshot" })); - expect(restored.getText("text").toString()).toBe(targetText.toString()); - }); - test("rejects duplicate text IDs before changing the document", () => { - const source = new LoroDoc(); - source.setPeerId(8); - source.getText("text").insert(0, "x".repeat(64)); - source.commit(); - const malformed = withDuplicateTextId(decodeFastSnapshot(source.export({ mode: "snapshot" }))); - const target = new LoroDoc(); - const events = []; - target.subscribe((event) => events.push(event)); - expect(() => target.import(malformed)).toThrow(/duplicate sequence IDs/u); - expect(events).toEqual([]); - expect(target.version().toJSON()).toEqual(new Map()); - expect(target.toJSON()).toEqual({}); - }); - test("rejects duplicate text IDs before installing shallow metadata", () => { - const source = new LoroDoc(); - source.setPeerId(9); - source.getText("text").insert(0, "x".repeat(64)); - source.commit(); - const shallowRoot = source.frontiers(); - source.getMap("meta").set("ready", true); - source.commit(); - const malformed = withDuplicateTextId(decodeFastSnapshot(source.export({ mode: "shallow-snapshot", frontiers: shallowRoot }))); - const target = new LoroDoc(); - const events = []; - target.subscribe((event) => events.push(event)); - expect(() => target.import(malformed)).toThrow(/duplicate sequence IDs/u); - expect(events).toEqual([]); - expect(target.isShallow()).toBe(false); - expect(target.version().toJSON()).toEqual(new Map()); - expect(target.toJSON()).toEqual({}); - const concurrent = new LoroDoc(); - concurrent.setPeerId(10); - concurrent.getMap("concurrent").set("value", true); - const batchTarget = new LoroDoc(); - const batchEvents = []; - batchTarget.subscribe((event) => batchEvents.push(event)); - expect(() => batchTarget.importBatch([malformed, concurrent.export({ mode: "update" })])).toThrow(/duplicate sequence IDs/u); - expect(batchEvents).toEqual([]); - expect(batchTarget.isShallow()).toBe(false); - expect(batchTarget.version().toJSON()).toEqual(new Map()); - expect(batchTarget.toJSON()).toEqual({}); - }); - test("keeps snapshot history lazy and owns the deferred bytes", () => { - const source = new LoroDoc(); - source.setPeerId(7); - source.getText("text").insert(0, "snapshot"); - source.getMap("meta").set("ready", true); - source.commit({ message: "seed" }); - const expectedVersion = source.version().toJSON(); - const expectedFrontiers = source.frontiers(); - const expectedChanges = source.getAllChanges(); - const snapshot = source.export({ mode: "snapshot" }); - const target = new LoroDoc(); - const events = []; - target.subscribe((event) => events.push(event)); - const status = target.import(snapshot); - expect(target.toJSON()).toEqual(source.toJSON()); - expect(target.version().toJSON()).toEqual(expectedVersion); - expect(target.frontiers()).toEqual(expectedFrontiers); - expect(target.opCount()).toBe(source.opCount()); - expect(status).toEqual({ - success: new Map([["7", { start: 0, end: 9 }]]), - pending: null, - }); - expect(events).toHaveLength(1); - expect(events[0]).toMatchObject({ by: "import", from: [], to: expectedFrontiers }); - snapshot.fill(0); - expect(target.getAllChanges()).toEqual(expectedChanges); - expect(target.changeCount()).toBe(source.changeCount()); - }); - test("materializes a snapshot before local edits without repeating first-peer events", () => { - const source = new LoroDoc(); - source.setPeerId(7); - source.getText("text").insert(0, "a"); - source.commit(); - const target = new LoroDoc(); - target.setPeerId(7); - target.import(source.export({ mode: "snapshot" })); - const firstPeers = []; - target.subscribeFirstCommitFromPeer(({ peer }) => firstPeers.push(peer)); - target.getText("text").push("b"); - target.commit(); - expect(firstPeers).toEqual([]); - expect(target.getText("text").toString()).toBe("ab"); - expect(target.version().get("7")).toBe(2); - }); - test("materializes a lazy snapshot before importing a later update", () => { - const source = new LoroDoc(); - source.setPeerId(9); - source.getText("text").insert(0, "a"); - source.commit(); - const snapshotVersion = source.version(); - const snapshot = source.export({ mode: "snapshot" }); - source.getText("text").push("b"); - source.commit(); - const update = source.export({ mode: "update", from: snapshotVersion }); - const target = LoroDoc.fromSnapshot(snapshot); - const status = target.import(update); - expect(status).toEqual({ - success: new Map([["9", { start: 1, end: 2 }]]), - pending: null, - }); - expect(target.toJSON()).toEqual({ text: "ab" }); - expect(target.getAllChanges()).toEqual(source.getAllChanges()); - }); - test("keeps deferred history atomic when snapshot metadata is inconsistent", () => { - const source = new LoroDoc(); - source.setPeerId(11); - source.getText("text").insert(0, "seed"); - source.commit(); - const snapshot = decodeFastSnapshot(source.export({ mode: "snapshot" })); - const oplog = decodeSstable(snapshot.oplog).map((entry) => { - if (entry.key.length !== 2 || entry.key[0] !== 0x76 || entry.key[1] !== 0x76) { - return entry; - } - const version = decodePostcardVersionVector(entry.value).map((id) => ({ - ...id, - counter: id.counter + 1, - })); - return { ...entry, value: encodePostcardVersionVector(version) }; - }); - const malformed = encodeFastSnapshot({ - ...snapshot, - oplog: encodeSstable(oplog, { compression: "none" }), - }); - const target = new LoroDoc(); - target.import(malformed); - expect(target.toJSON()).toEqual({ text: "seed" }); - expect(() => target.changeCount()).toThrow(/version does not match/u); - expect(target.toJSON()).toEqual({ text: "seed" }); - expect(() => target.changeCount()).toThrow(/version does not match/u); - }); - test("rejects a corrupt deferred frontier block before changing the document", () => { - const source = new LoroDoc(); - source.setPeerId(12); - source.getText("text").insert(0, "seed"); - source.commit(); - const snapshot = decodeFastSnapshot(source.export({ mode: "snapshot" })); - const oplog = decodeSstable(snapshot.oplog).map((entry) => entry.key.length === 12 ? { ...entry, value: Uint8Array.of(0xff) } : entry); - const malformed = encodeFastSnapshot({ - ...snapshot, - oplog: encodeSstable(oplog, { compression: "none" }), - }); - const target = new LoroDoc(); - const events = []; - target.subscribe((event) => events.push(event)); - expect(() => target.import(malformed)).toThrow(/unexpected end|change block/u); - expect(events).toEqual([]); - expect(target.version().toJSON()).toEqual(new Map()); - expect(target.toJSON()).toEqual({}); - }); - test("owns shallow root bytes until deferred history is materialized", () => { - const source = new LoroDoc(); - source.setPeerId(13); - const original = new Uint8Array(64); - let random = 305419896; - for (let index = 0; index < original.length; index += 1) { - random ^= random << 13; - random ^= random >>> 17; - random ^= random << 5; - original[index] = random & 0xff; - } - source.getMap("map").set("bytes", original); - source.commit(); - const shallowRoot = source.frontiers(); - source.getMap("map").set("bytes", Uint8Array.of(9)); - source.commit(); - const bytes = source.export({ - mode: "shallow-snapshot", - frontiers: shallowRoot, - }); - const target = LoroDoc.fromSnapshot(bytes); - expect(target.getMap("map").get("bytes")).toEqual(Uint8Array.of(9)); - bytes.fill(0); - target.checkout(shallowRoot); - expect(target.getMap("map").get("bytes")).toEqual(original); - }); - test("does not install shallow metadata before latest state decoding succeeds", () => { - const source = new LoroDoc(); - source.setPeerId(14); - source.getText("text").insert(0, "a"); - source.commit(); - const shallowRoot = source.frontiers(); - source.getText("text").push("b"); - source.commit(); - const snapshot = decodeFastSnapshot(source.export({ mode: "shallow-snapshot", frontiers: shallowRoot })); - const malformed = encodeFastSnapshot({ - ...snapshot, - state: Uint8Array.of(0xff), - }); - const target = new LoroDoc(); - expect(() => target.import(malformed)).toThrow(/SSTable is too short/u); - expect(target.isShallow()).toBe(false); - expect(target.version().toJSON()).toEqual(new Map()); - expect(target.toJSON()).toEqual({}); - }); - test("exports and imports shallow snapshots at a partial change boundary", () => { - const source = new LoroDoc(); - source.setPeerId(3); - source.getText("text").insert(0, "01234"); - source.commit(); - const shallowRoot = source.frontiers(); - source.getText("text").push("56789"); - source.getMap("meta").set("ready", true); - source.commit(); - const snapshot = source.export({ - mode: "shallow-snapshot", - frontiers: shallowRoot, - }); - const target = LoroDoc.fromSnapshot(snapshot); - expect(target.toJSON()).toEqual(source.toJSON()); - expect(target.isShallow()).toBe(true); - expect(target.shallowSinceFrontiers()).toEqual(shallowRoot); - expect(target.shallowSinceVV().toJSON()).toEqual(new Map([["3", 4]])); - expect(() => target.checkout([])).toThrow(/before the shallow history root/u); - target.checkout(shallowRoot); - expect(target.toJSON()).toEqual({ text: "01234", meta: {} }); - target.checkoutToLatest(); - expect(target.toJSON()).toEqual(source.toJSON()); - const resnapshot = target.export({ mode: "snapshot" }); - const restored = LoroDoc.fromSnapshot(resnapshot); - expect(restored.isShallow()).toBe(true); - expect(restored.toJSON()).toEqual(source.toJSON()); - }); - test("preserves rich-text attributes in snapshots", () => { - const source = new LoroDoc(); - source.setPeerId(7); - const text = source.getText("text"); - text.insert(0, "a😀bc"); - text.mark({ start: 1, end: 3 }, "bold", true); - source.commit(); - const target = LoroDoc.fromSnapshot(source.export({ mode: "snapshot" })); - expect(target.getText("text").toDelta()).toEqual([ - { insert: "a" }, - { insert: "😀", attributes: { bold: true } }, - { insert: "bc" }, - ]); - }); - test("uses compatible text-style expansion rules", () => { - const doc = new LoroDoc(); - const text = doc.getText("text"); - text.insert(0, "hi"); - text.mark({ start: 0, end: 2 }, "bold", true); - text.insert(2, "!"); - text.insert(0, "("); - expect(text.toDelta()).toEqual([ - { insert: "(" }, - { insert: "hi!", attributes: { bold: true } }, - ]); - const link = doc.getText("link"); - link.insert(0, "x"); - link.mark({ start: 0, end: 1 }, "link", "url"); - link.insert(1, "y"); - expect(link.toDelta()).toEqual([ - { insert: "x", attributes: { link: "url" } }, - { insert: "y" }, - ]); - }); - test("matches sorted map and text edge contracts", () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - map.set("z", 1); - map.set("a", 2); - expect(map.entries()).toEqual([ - ["a", 2], - ["z", 1], - ]); - const text = doc.getText("text"); - text.insert(0, "x"); - expect(text.convertPos(0, "invalid", "unicode")).toBeUndefined(); - expect(() => text.mark({ start: 0, end: 1 }, "not-configured", true)).toThrow(/not configured/u); - const detached = new LoroText(); - detached.insert(0, "x"); - expect(() => detached.mark({ start: 0, end: 1 }, "not-configured", true)).not.toThrow(); - }); - test("tracks stable cursors and replaces deleted anchors", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "123"); - const cursor = text.getCursor(0, 0); - const end = text.getCursor(10, 0); - expect(cursor.kind()).toBe("Cursor"); - expect(cursor.containerId()).toBe(text.id); - expect(cursor.pos()).toEqual({ peer: "1", counter: 0 }); - expect(end.side()).toBe(1); - expect(doc.getCursorPos(cursor)).toEqual({ offset: 0, side: 0 }); - text.insert(0, "abc"); - expect(doc.getCursorPos(cursor)).toEqual({ offset: 3, side: 0 }); - const decoded = Cursor.decode(cursor.encode()); - const remote = new LoroDoc(); - remote.import(doc.export({ mode: "update" })); - expect(remote.getCursorPos(decoded)).toEqual({ offset: 3, side: 0 }); - text.delete(3, 1); - doc.commit(); - const result = doc.getCursorPos(cursor); - expect(result.offset).toBe(3); - expect(result.side).toBe(-1); - expect(result.update?.pos()).toEqual({ peer: "1", counter: 1 }); - expect(result.update?.side()).toBe(-1); - }); - test("copies detached child containers and exposes their attached handlers", () => { - const child = new LoroMap(); - child.set("ready", true); - const nestedList = new LoroList(); - nestedList.push(1); - child.setContainer("items", nestedList); - const doc = new LoroDoc(); - doc.setPeerId(3); - const attachedChild = doc.getMap("root").setContainer("child", child); - expect(doc.toJSON()).toEqual({ root: { child: { ready: true, items: [1] } } }); - expect(child.isAttached()).toBe(false); - expect(child.getAttached()).toBe(attachedChild); - expect(nestedList.isAttached()).toBe(false); - expect(nestedList.getAttached()?.isAttached()).toBe(true); - child.set("detached-only", true); - attachedChild.set("attached-only", true); - expect(doc.toJSON()).toEqual({ - root: { child: { ready: true, items: [1], "attached-only": true } }, - }); - }); - test("converges concurrent edits in deterministic mergeable children", () => { - const left = new LoroDoc(); - left.setPeerId(1); - const right = new LoroDoc(); - right.setPeerId(2); - const leftCounter = left.getMap("state").ensureMergeableCounter("revision"); - const rightCounter = right.getMap("state").ensureMergeableCounter("revision"); - leftCounter.increment(1); - rightCounter.increment(1); - const leftUpdate = left.export({ mode: "update" }); - const rightUpdate = right.export({ mode: "update" }); - left.import(rightUpdate); - right.import(leftUpdate); - expect(leftCounter.id).toBe(rightCounter.id); - expect(leftCounter.value).toBe(2); - expect(rightCounter.value).toBe(2); - expect(left.toJSON()).toEqual({ state: { revision: 2 } }); - expect(right.toJSON()).toEqual(left.toJSON()); - }); - test("integrates update batches once in arbitrary blob order", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const text = source.getText("text"); - text.push("a"); - source.commit(); - const firstVersion = source.oplogVersion(); - const first = source.export({ mode: "update" }); - text.push("b"); - source.commit(); - const second = source.export({ mode: "update", from: firstVersion }); - const target = new LoroDoc(); - let events = 0; - target.subscribe(() => { - events += 1; - }); - const status = target.importBatch([second, first]); - expect(target.toJSON()).toEqual({ text: "ab" }); - expect(events).toBe(1); - expect(status.success.get("1")).toEqual({ start: 0, end: 2 }); - expect(status.pending).toBeNull(); - }); - test("integrates mixed snapshot and update batches once", () => { - const source = new LoroDoc(); - source.setPeerId(1); - const text = source.getText("text"); - text.push("a"); - source.commit(); - const firstSnapshot = source.export({ mode: "snapshot" }); - text.push("b"); - source.commit(); - const secondVersion = source.oplogVersion(); - const secondSnapshot = source.export({ mode: "snapshot" }); - text.push("c"); - source.commit(); - const update = source.export({ mode: "update", from: secondVersion }); - const target = new LoroDoc(); - const events = []; - target.subscribe((event) => events.push(event)); - const status = target.importBatch([update, firstSnapshot, secondSnapshot]); - expect(target.toJSON()).toEqual({ text: "abc" }); - expect(events).toHaveLength(1); - expect(events[0].events).toEqual([ - { - target: "cid:root-text:Text", - path: ["text"], - diff: { type: "text", diff: [{ insert: "abc" }] }, - }, - ]); - expect(status.success.get("1")).toEqual({ start: 0, end: 3 }); - expect(status.pending).toBeNull(); - }); - test("keeps mergeable ensure idempotent and guards non-mergeable values", () => { - const doc = new LoroDoc(); - const root = doc.getMap("state"); - const first = root.ensureMergeableMap("profile"); - doc.commit(); - const opCount = doc.opCount(); - expect(root.ensureMergeableMap("profile")).toBe(first); - doc.commit(); - expect(doc.opCount()).toBe(opCount); - root.set("scalar", 1); - expect(() => root.ensureMergeableList("scalar")).toThrow(/non-mergeable value/u); - expect(root.get("scalar")).toBe(1); - root.setContainer("regular", new LoroMap()); - expect(() => root.ensureMergeableMap("regular")).toThrow(/non-mergeable value/u); - }); - test("does not record semantic no-op edits", () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - map.set("value", { nested: [1, true, null] }); - const movable = doc.getMovableList("movable"); - movable.push("same"); - doc.commit(); - const opCount = doc.opCount(); - map.set("value", { nested: [1, true, null] }); - map.delete("missing"); - movable.set(0, "same"); - doc.getCounter("counter").increment(0); - doc.commit(); - expect(doc.opCount()).toBe(opCount); - }); - test("does not confuse empty value shapes when detecting no-op edits", () => { - const doc = new LoroDoc(); - const map = doc.getMap("map"); - const movable = doc.getMovableList("movable"); - map.set("value", []); - movable.push([]); - doc.commit(); - const opCount = doc.opCount(); - map.set("value", {}); - movable.set(0, {}); - doc.commit(); - map.set("value", new Uint8Array()); - movable.set(0, new Uint8Array()); - doc.commit(); - expect(map.get("value")).toEqual(new Uint8Array()); - expect(movable.get(0)).toEqual(new Uint8Array()); - expect(doc.opCount()).toBe(opCount + 4); - }); - test("resurfaces preserved state and switches mergeable kinds", () => { - const doc = new LoroDoc(); - const root = doc.getMap("state"); - const text = root.ensureMergeableText("field"); - text.insert(0, "hello"); - doc.commit(); - root.delete("field"); - doc.commit(); - expect(root.get("field")).toBeUndefined(); - expect(text.isDeleted()).toBe(true); - const restored = root.ensureMergeableText("field"); - expect(restored).toBe(text); - expect(restored.toString()).toBe("hello"); - const map = root.ensureMergeableMap("field"); - map.set("value", 1); - expect(doc.toJSON()).toEqual({ state: { field: { value: 1 } } }); - expect(root.ensureMergeableText("field").toString()).toBe("hello"); - }); - test("uses flattened mergeable ids and logical event paths", () => { - const doc = new LoroDoc(); - const root = doc.getMap("state"); - const profile = root.ensureMergeableMap("profile"); - const counter = profile.ensureMergeableCounter("revision"); - const batches = []; - root.subscribe((event) => batches.push(event)); - counter.increment(3); - doc.commit(); - expect(profile.id).toBe("cid:root-🤝:$state>profile:Map"); - expect(counter.id).toBe("cid:root-🤝:$state>profile>revision:Counter"); - expect(doc.getContainerById(counter.id)).toBe(counter); - expect(batches.at(-1)?.events.find((event) => event.target === counter.id)?.path).toEqual(["state", "profile", "revision"]); - }); - test("distinguishes existing roots, normal children, and mergeable children", () => { - const doc = new LoroDoc(); - expect(doc.hasContainer("cid:root-new:Text")).toBe(true); - expect(doc.getContainerById("cid:root-new:Text")?.kind()).toBe("Text"); - expect(doc.hasContainer("cid:0@99:Text")).toBe(false); - expect(() => doc.getText("cid:0@99:Text")).toThrow(/does not exist/); - const root = doc.getMap("state"); - const empty = root.ensureMergeableText("empty"); - expect(doc.hasContainer(empty.id)).toBe(true); - root.delete("empty"); - expect(doc.hasContainer(empty.id)).toBe(false); - const preserved = root.ensureMergeableText("preserved"); - preserved.insert(0, "kept"); - doc.commit(); - root.delete("preserved"); - doc.commit(); - expect(doc.hasContainer(preserved.id)).toBe(true); - expect(preserved.isDeleted()).toBe(true); - }); - test("passes live containers through toJsonWithReplacer", () => { - const doc = new LoroDoc(); - const root = doc.getMap("root"); - root.set("count", 1); - root.set("drop", true); - const body = root.setContainer("body", new LoroText()); - body.insert(0, "hello"); - const seen = []; - expect(doc.toJsonWithReplacer((key, value) => { - if (value instanceof LoroMap || value instanceof LoroText) { - seen.push(`${String(key)}:${value.kind()}`); - } - if (value instanceof LoroText) - return value.toString().toUpperCase(); - if (key === "count") - return 2; - if (key === "drop") - return undefined; - return value; - })).toEqual({ root: { count: 2, body: "HELLO" } }); - expect(seen).toEqual(["root:Map", "body:Text"]); - expect(() => doc.toJsonWithReplacer((_key, value) => (value instanceof LoroText ? root : value))).toThrow(/different container/u); - }); - test("iterates text by contiguous operation runs", () => { - const left = new LoroDoc(); - left.setPeerId(1); - left.getText("text").insert(0, "Hello"); - left.commit(); - const right = left.fork(); - right.setPeerId(2); - right.getText("text").insert(3, " "); - left.import(right.export({ mode: "update", from: left.oplogVersion() })); - const chunks = []; - left.getText("text").iter((chunk) => { - chunks.push(chunk); - }); - expect(chunks).toEqual(["Hel", " ", "lo"]); - const firstChunk = []; - left.getText("text").iter((chunk) => { - firstChunk.push(chunk); - return false; - }); - expect(firstChunk).toEqual(["Hel"]); - expect(left.getText("text").sliceDeltaUtf8(3, 4)).toEqual([{ insert: " " }]); - expect(left.getText("text").getEditorOf(3)).toBe("2"); - }); - test("guards tree cycles and reports move and fractional-index state", () => { - const doc = new LoroDoc(); - doc.setPeerId(7); - const tree = doc.getTree("tree"); - const root = tree.createNode(); - const child = root.createNode(); - const creation = child.creationId(); - expect(child.creator()).toBe("7"); - expect(child.getLastMoveId()).toEqual(creation); - expect(child.fractionalIndex()).toBeTypeOf("string"); - expect(() => root.move(child)).toThrow(/descendant/u); - child.move(undefined, 0); - expect(child.getLastMoveId()).not.toEqual(creation); - tree.disableFractionalIndex(); - expect(tree.isFractionalIndexEnabled()).toBe(false); - expect(child.fractionalIndex()).toBeUndefined(); - tree.enableFractionalIndex(); - expect(tree.isFractionalIndexEnabled()).toBe(true); - expect(() => tree.enableFractionalIndex(256)).toThrow(/unsigned byte/u); - const lastMoveBeforeDelete = child.getLastMoveId(); - tree.delete(child.id); - expect(tree.isNodeDeleted(child.id)).toBe(true); - expect(() => tree.isNodeDeleted("999@999")).toThrow(/does not exist/u); - expect(tree.getNodes({ withDeleted: true })).toHaveLength(2); - expect(root.toJSON()).toMatchObject({ parent: undefined, fractionalIndex: "80" }); - expect(child.parent()?.id).toBe("2147483647@18446744073709551615"); - expect(child.index()).toBeUndefined(); - expect(child.fractionalIndex()).toBeUndefined(); - expect(child.getLastMoveId()).not.toEqual(lastMoveBeforeDelete); - expect(child.toJSON()).toMatchObject({ - parent: "2147483647@18446744073709551615", - index: 0, - fractionalIndex: "80", - }); - }); - test("keeps deleted tree subtrees out of the live forest and snapshots them", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const tree = doc.getTree("tree"); - tree.enableFractionalIndex(0); - const liveRoot = tree.createNode(); - const deletedRoot = tree.createNode(); - const deletedChild = deletedRoot.createNode(); - expect(deletedRoot.toJSON().fractionalIndex).toBe("8180"); - tree.delete(deletedRoot.id); - expect(tree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); - expect(tree.getNodes({ withDeleted: true }).map((node) => node.id)).toEqual([ - liveRoot.id, - deletedRoot.id, - deletedChild.id, - ]); - expect(tree.isNodeDeleted(deletedChild.id)).toBe(true); - expect(deletedChild.isDeleted()).toBe(true); - expect(deletedChild.data.isDeleted()).toBe(true); - expect(deletedRoot.toJSON()).toMatchObject({ - parent: "2147483647@18446744073709551615", - index: 0, - fractionalIndex: "80", - }); - doc.commit(); - const restored = new LoroDoc(); - restored.import(doc.export({ mode: "snapshot" })); - const restoredTree = restored.getTree("tree"); - expect(restoredTree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); - expect(restoredTree.isNodeDeleted(deletedRoot.id)).toBe(true); - expect(restoredTree.isNodeDeleted(deletedChild.id)).toBe(true); - expect(restoredTree.getNodeByID(deletedRoot.id)?.toJSON()).toMatchObject({ - parent: "2147483647@18446744073709551615", - index: 0, - fractionalIndex: "80", - }); - }); - test("round-trips frontiers and reports a package version", () => { - const doc = new LoroDoc(); - doc.setPeerId(11); - doc.getText("text").insert(0, "hello"); - doc.commit(); - const frontiers = doc.frontiers(); - expect(decodeFrontiers(encodeFrontiers(frontiers))).toEqual(frontiers); - expect(LORO_VERSION()).toMatch(/^\d+\.\d+\.\d+(?:[-+].+)?$/u); - }); - test("synchronizes postcard-compatible awareness state", () => { - const a = new AwarenessWasm("1", 30000); - const b = new AwarenessWasm("2", 30000); - a.setLocalState({ cursor: 3, bytes: Uint8Array.of(1, 2, 3) }); - expect(b.apply(a.encode(["1"]))).toEqual({ added: ["1"], updated: [] }); - expect(b.getState("1")).toEqual({ cursor: 3, bytes: Uint8Array.of(1, 2, 3) }); - expect(b.peer()).toBe("2"); - expect(b.length()).toBe(1); - const old = a.encodeAll(); - a.setLocalState({ cursor: 4 }); - const current = a.encodeAll(); - expect(b.apply(current)).toEqual({ added: [], updated: ["1"] }); - expect(b.apply(old)).toEqual({ added: [], updated: [] }); - expect(b.getState("1")).toEqual({ cursor: 4 }); - expect(() => b.apply(Uint8Array.of(0xff))).toThrow(/Failed to decode awareness/u); - const wrapped = new Awareness("3", 30000); - const events = []; - wrapped.addListener((update, origin) => events.push([update, origin])); - wrapped.setLocalState("online"); - expect(wrapped.getLocalState()).toBe("online"); - expect(events).toEqual([[{ updated: [], added: ["3"], removed: [] }, "local"]]); - wrapped.destroy(); - }); - test("synchronizes typed ephemeral state and deletion tombstones", async () => { - const a = new EphemeralStoreWasm(30000); - const b = new EphemeralStoreWasm(30000); - const localUpdates = []; - const events = []; - a.subscribeLocalUpdates((bytes) => localUpdates.push(bytes)); - b.subscribe((event) => events.push(event)); - a.set("cursor", { x: 1, bytes: Uint8Array.of(4, 5) }); - b.apply(localUpdates.at(-1)); - expect(b.getAllStates()).toEqual({ - cursor: { x: 1, bytes: Uint8Array.of(4, 5) }, - }); - expect(events.at(-1)).toEqual({ - by: "import", - added: ["cursor"], - updated: [], - removed: [], - }); - await new Promise((resolve) => setTimeout(resolve, 2)); - a.delete("cursor"); - b.apply(localUpdates.at(-1)); - expect(b.get("cursor")).toBeUndefined(); - expect(events.at(-1)).toEqual({ - by: "import", - added: [], - updated: [], - removed: ["cursor"], - }); - const typed = new EphemeralStore(30000); - typed.set("name", "Ada"); - typed.set("position", 2); - expect(typed.getAllStates()).toEqual({ name: "Ada", position: 2 }); - typed.destroy(); - }); - test("queues nested ephemeral updates until the current callback returns", () => { - const store = new EphemeralStore(30000); - let calls = 0; - store.subscribe(() => { - if (calls === 0) - store.set("a", 2); - calls += 1; - }); - store.set("a", 1); - store.set("b", 2); - store.set("c", 3); - expect(calls).toBe(4); - expect(store.getAllStates()).toEqual({ a: 2, b: 2, c: 3 }); - store.destroy(); - }); - test("provides compatible detached classes and version vectors", () => { - expect(new LoroMap().kind()).toBe("Map"); - expect(new LoroList().kind()).toBe("List"); - expect(new LoroMovableList().kind()).toBe("MovableList"); - expect(new LoroText().kind()).toBe("Text"); - expect(new LoroCounter().kind()).toBe("Counter"); - const version = new VersionVector(new Map([["1", 2]])); - version.setLast({ peer: "2", counter: 3 }); - expect(VersionVector.decode(version.encode()).toJSON()).toEqual(new Map([ - ["1", 2], - ["2", 4], - ])); - }); - test("loads the deep value from a current Rust snapshot", () => { - const doc = LoroDoc.fromSnapshot(fixture("snapshot.blob")); - const expected = JSON.parse(readFileSync(new URL("./fixtures/rust/snapshot.deep.json", import.meta.url), "utf8")); - const actual = doc.toJSON(); - const actualMap = actual.map; - const expectedMap = expected.map; - const actualChildMap = actualMap.child_map; - expect(actualChildMap.a).toBe(1); - expect(actualChildMap.t).toBe("inn"); - expect(actualChildMap.v1476140860).toEqual(Uint8Array.of(0, 1, 2, 46)); - expect(actualMap.child_list).toEqual(expectedMap.child_list); - expect(actualMap.child_mlist).toEqual(expectedMap.child_mlist); - expect(actual.list).toEqual(expected.list); - expect(actual.mlist).toEqual(expected.mlist); - expect(actual.text).toEqual(expected.text); - expect(actual.tree.length).toBe(expected.tree.length); - const version = doc.version().toJSON(); - expect(doc.changeCount()).toBeGreaterThan(0); - expect(doc.version().toJSON()).toEqual(version); - expect(doc.toJSON()).toEqual(actual); - }); -}); diff --git a/loro-js/tests/rust-interop.test.js b/loro-js/tests/rust-interop.test.js deleted file mode 100644 index c14dade47..000000000 --- a/loro-js/tests/rust-interop.test.js +++ /dev/null @@ -1,176 +0,0 @@ -import { readFileSync } from "node:fs"; -import { describe, expect, test } from "vitest"; -import { ContainerType, EncodeMode, bytesEqual, decodeChangeBlockKey, decodeChangeBlock, decodeChangeKeys, decodeChangeValueContent, decodeColumnarVecMaybeWrapped, decodeContainerId, decodeContainerArena, decodeDocument, decodeDeleteStartIds, decodeEncodedChangeBlock, decodeEncodedOperations, decodeFastSnapshot, decodeFastUpdates, decodeDeltaRleI32, decodeDeltaRleU32, decodeChangesHeader, decodeChangesMetadata, decodePositionArena, decodePostcardFrontiers, decodePostcardVersionVector, decodeRleU8, decodeRleU32, decodeSstable, decodeStateSnapshotStore, encodeEncodedChangeBlock, encodeChangeBlock, encodeFastSnapshot, encodeFastUpdates, encodeSstable, encodeStateSnapshotStore, } from "../src/codec/index"; -const fixture = (name) => new Uint8Array(readFileSync(new URL(`./fixtures/rust/${name}`, import.meta.url))); -const textEncoder = new TextEncoder(); -const VV_KEY = textEncoder.encode("vv"); -const FRONTIERS_KEY = textEncoder.encode("fr"); -describe("current Rust interoperability", () => { - test("reads and losslessly reframes a Rust FastUpdates blob", () => { - const bytes = fixture("updates.blob"); - expect(decodeDocument(bytes).mode).toBe(EncodeMode.FastUpdates); - const blocks = decodeFastUpdates(bytes); - expect(blocks.length).toBeGreaterThan(1); - expect(encodeFastUpdates(blocks)).toEqual(bytes); - }); - test("reads every Rust encoded change block and its operation columns", () => { - const blocks = decodeFastUpdates(fixture("updates.blob")); - let operationCount = 0; - for (const bytes of blocks) { - const block = decodeEncodedChangeBlock(bytes); - expect(encodeEncodedChangeBlock(block)).toEqual(bytes); - expect(block.changeCount).toBeGreaterThan(0); - const columns = decodeColumnarVecMaybeWrapped(block.operations); - expect(columns).toHaveLength(4); - const containerIndices = decodeDeltaRleU32(columns[0]); - const props = decodeDeltaRleI32(columns[1]); - const valueTypes = decodeRleU8(columns[2]); - const lengths = decodeRleU32(columns[3]); - expect(props).toHaveLength(containerIndices.length); - expect(valueTypes).toHaveLength(containerIndices.length); - expect(lengths).toHaveLength(containerIndices.length); - operationCount += lengths.length; - } - expect(operationCount).toBeGreaterThan(40); - }); - test("decodes every semantic table in the Rust change blocks", () => { - const blocks = decodeFastUpdates(fixture("updates.blob")); - let changeCount = 0; - let nestedContainerCount = 0; - const valueTypes = new Set(); - for (const bytes of blocks) { - const block = decodeEncodedChangeBlock(bytes); - const header = decodeChangesHeader(block.header, { - changeCount: block.changeCount, - counterStart: block.counterStart, - counterLength: block.counterLength, - lamportStart: block.lamportStart, - lamportLength: block.lamportLength, - }); - const metadata = decodeChangesMetadata(block.changeMetadata, block.changeCount); - const keys = decodeChangeKeys(block.keys); - const containers = decodeContainerArena(block.containerIds, header.peers, keys); - const positions = decodePositionArena(block.positions); - const operations = decodeEncodedOperations(block.operations); - const deletes = decodeDeleteStartIds(block.deleteStartIds); - let remainingValues = block.values; - for (const operation of operations) { - const [value, remaining] = decodeChangeValueContent(operation.valueType, remainingValues); - valueTypes.add(value.type); - remainingValues = remaining; - } - expect(header.lengths).toHaveLength(block.changeCount); - expect(header.dependencies).toHaveLength(block.changeCount); - expect(metadata.timestamps).toHaveLength(block.changeCount); - expect(metadata.commitMessages).toHaveLength(block.changeCount); - expect(operations.reduce((sum, operation) => sum + operation.length, 0)).toBe(block.counterLength); - expect(operations.every((operation) => operation.containerIndex < containers.length)).toBe(true); - expect(deletes.every((entry) => entry.peerIndex < BigInt(header.peers.length))).toBe(true); - expect(remainingValues).toHaveLength(0); - changeCount += block.changeCount; - nestedContainerCount += containers.filter((container) => container.kind === "normal").length; - expect(positions.length).toBeGreaterThanOrEqual(0); - } - expect(changeCount).toBeGreaterThanOrEqual(5); - expect(nestedContainerCount).toBeGreaterThan(0); - expect([...valueTypes]).toEqual(expect.arrayContaining(["loro-value", "string", "raw-tree-move"])); - }); - test("semantically decodes and re-encodes every Rust change block", () => { - const operationTypes = new Set(); - let operationCount = 0; - for (const bytes of decodeFastUpdates(fixture("updates.blob"))) { - const decoded = decodeChangeBlock(bytes); - for (const change of decoded.changes) { - for (const operation of change.operations) { - operationTypes.add(operation.content.type); - operationCount += 1; - } - } - const decodedAgain = decodeChangeBlock(encodeChangeBlock(decoded)); - expect(decodedAgain.changes).toEqual(decoded.changes); - } - expect(operationCount).toBeGreaterThan(40); - expect([...operationTypes]).toEqual(expect.arrayContaining([ - "map-insert", - "text-insert", - "list-insert", - "movable-list-insert", - "tree-create", - ])); - }); - test("reads and losslessly reframes a Rust FastSnapshot blob", () => { - const bytes = fixture("snapshot.blob"); - expect(decodeDocument(bytes).mode).toBe(EncodeMode.FastSnapshot); - const snapshot = decodeFastSnapshot(bytes); - expect(snapshot.oplog.length).toBeGreaterThan(0); - expect(snapshot.state.length).toBeGreaterThan(0); - expect(snapshot.shallowRootState).toHaveLength(0); - expect(encodeFastSnapshot(snapshot)).toEqual(bytes); - }); - test("decodes every Rust ChangeStore and state SSTable block", () => { - const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); - const history = decodeSstable(snapshot.oplog); - const state = decodeSstable(snapshot.state); - const utf8 = new TextDecoder(); - const historyKeys = history.map((entry) => utf8.decode(entry.key)); - expect(historyKeys).toContain("vv"); - expect(historyKeys).toContain("fr"); - expect(history.length).toBeGreaterThan(2); - expect(state.length).toBeGreaterThan(5); - }); - test("decodes semantic keys and versions from a Rust snapshot", () => { - const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); - const history = decodeSstable(snapshot.oplog); - const versionEntry = history.find((entry) => bytesEqual(entry.key, VV_KEY)); - const frontiersEntry = history.find((entry) => bytesEqual(entry.key, FRONTIERS_KEY)); - expect(versionEntry).toBeDefined(); - expect(frontiersEntry).toBeDefined(); - const version = decodePostcardVersionVector(versionEntry.value); - const frontiers = decodePostcardFrontiers(frontiersEntry.value); - expect(version.length).toBeGreaterThan(0); - expect(frontiers.length).toBeGreaterThan(0); - const blockIds = history - .filter((entry) => !bytesEqual(entry.key, VV_KEY) && !bytesEqual(entry.key, FRONTIERS_KEY)) - .map((entry) => decodeChangeBlockKey(entry.key)); - expect(blockIds.length).toBeGreaterThan(0); - expect(blockIds.every((id) => id.counter >= 0)).toBe(true); - const containerIds = decodeSstable(snapshot.state) - .filter((entry) => !bytesEqual(entry.key, FRONTIERS_KEY)) - .map((entry) => decodeContainerId(entry.key)); - const rootNames = containerIds.flatMap((id) => (id.kind === "root" ? [id.name] : [])); - expect(rootNames).toEqual(expect.arrayContaining(["map", "list", "text", "mlist", "tree"])); - expect(containerIds.some((id) => id.kind === "normal")).toBe(true); - }); - test("semantically decodes and re-encodes every Rust container state", () => { - const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); - const store = decodeStateSnapshotStore(snapshot.state); - expect(store.kind).toBe("sstable"); - if (store.kind !== "sstable") { - return; - } - const kinds = new Set(store.containers.map(({ wrapper }) => wrapper.state.kind)); - expect([...kinds]).toEqual(expect.arrayContaining([ - ContainerType.Map, - ContainerType.List, - ContainerType.Text, - ContainerType.Tree, - ContainerType.MovableList, - ])); - expect(store.containers.some(({ id }) => id.kind === "normal")).toBe(true); - const encoded = encodeStateSnapshotStore(store, { compression: "none" }); - expect(decodeStateSnapshotStore(encoded)).toEqual(store); - }); - test("rewrites Rust SSTables into independently decodable TypeScript SSTables", () => { - const snapshot = decodeFastSnapshot(fixture("snapshot.blob")); - for (const table of [snapshot.oplog, snapshot.state]) { - const entries = decodeSstable(table); - const rewritten = encodeSstable(entries, { compression: "none" }); - const decodedAgain = decodeSstable(rewritten); - expect(decodedAgain).toHaveLength(entries.length); - for (let index = 0; index < entries.length; index += 1) { - expect(bytesEqual(decodedAgain[index].key, entries[index].key)).toBe(true); - expect(bytesEqual(decodedAgain[index].value, entries[index].value)).toBe(true); - } - } - }); -}); diff --git a/loro-js/tests/semantic-codecs.test.js b/loro-js/tests/semantic-codecs.test.js deleted file mode 100644 index 1d8ec9391..000000000 --- a/loro-js/tests/semantic-codecs.test.js +++ /dev/null @@ -1,110 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { ContainerType, bytesToHex, decodeChangeBlockKey, decodeContainerId, decodePostcardContainerId, decodePostcardFrontiers, decodePostcardOptionalContainerId, decodePostcardValue, decodePostcardVersionVector, encodeChangeBlockKey, encodeContainerId, encodePostcardContainerId, encodePostcardFrontiers, encodePostcardOptionalContainerId, encodePostcardValue, encodePostcardVersionVector, } from "../src/codec/index"; -describe("IDs and container IDs", () => { - test("matches the Rust ChangeStore key layout", () => { - const id = { peer: 0x0102030405060708n, counter: -1 }; - const encoded = encodeChangeBlockKey(id); - expect(bytesToHex(encoded)).toBe("0102030405060708ffffffff"); - expect(decodeChangeBlockKey(encoded)).toEqual(id); - }); - test.each([ - [{ kind: "root", name: "root", containerType: ContainerType.Map }, "8004726f6f74"], - [ - { - kind: "normal", - peer: 7n, - counter: 42, - containerType: ContainerType.List, - }, - "0107000000000000002a000000", - ], - ])("matches the raw ContainerID layout", (id, expected) => { - const encoded = encodeContainerId(id); - expect(bytesToHex(encoded)).toBe(expected); - expect(decodeContainerId(encoded)).toEqual(id); - }); - test.each([ - [undefined, "00"], - [ - { kind: "root", name: "root", containerType: ContainerType.Map }, - "010004726f6f7401", - ], - [{ kind: "root", name: "t", containerType: ContainerType.Text }, "0100017400"], - [ - { - kind: "normal", - peer: 7n, - counter: 42, - containerType: ContainerType.List, - }, - "0101075402", - ], - ])("matches Rust postcard Option", (id, expected) => { - const encoded = encodePostcardOptionalContainerId(id); - expect(bytesToHex(encoded)).toBe(expected); - expect(decodePostcardOptionalContainerId(encoded)).toEqual(id); - }); - test("round trips postcard ContainerID without the option wrapper", () => { - const id = { - kind: "normal", - peer: 0xffffffffffffffffn, - counter: -2147483648, - containerType: ContainerType.Tree, - }; - expect(decodePostcardContainerId(encodePostcardContainerId(id))).toEqual(id); - }); -}); -describe("VersionVector and Frontiers", () => { - test("preserves VersionVector order", () => { - const version = [ - { peer: 7n, counter: 42 }, - { peer: 1n, counter: -1 }, - ]; - expect(decodePostcardVersionVector(encodePostcardVersionVector(version))).toEqual(version); - }); - test("canonically sorts Frontiers by peer then counter", () => { - const frontiers = [ - { peer: 7n, counter: 42 }, - { peer: 1n, counter: 0 }, - { peer: 7n, counter: 41 }, - { peer: 1n, counter: -1 }, - ]; - expect(decodePostcardFrontiers(encodePostcardFrontiers(frontiers))).toEqual([ - { peer: 1n, counter: -1 }, - { peer: 1n, counter: 0 }, - { peer: 7n, counter: 41 }, - { peer: 7n, counter: 42 }, - ]); - }); -}); -describe("postcard LoroValue", () => { - const values = [ - { type: "null" }, - { type: "bool", value: false }, - { type: "bool", value: true }, - { type: "double", value: -0 }, - { type: "i64", value: -1n }, - { type: "string", value: "hi 😀" }, - { type: "binary", value: Uint8Array.of(1, 2) }, - { - type: "list", - value: [{ type: "null" }, { type: "i64", value: 1n }], - }, - { - type: "map", - value: [ - ["a", { type: "null" }], - ["b", { type: "i64", value: 2n }], - ], - }, - { - type: "container", - value: { kind: "root", name: "x", containerType: ContainerType.Text }, - }, - ]; - test.each(values)("round trips $type", (value) => { - const decoded = decodePostcardValue(encodePostcardValue(value)); - expect(decoded).toEqual(value); - expect(value.type !== "double" || Object.is(decoded.value, -0)).toBe(true); - }); -}); diff --git a/loro-js/tests/serde-columnar.test.js b/loro-js/tests/serde-columnar.test.js deleted file mode 100644 index dcf522cac..000000000 --- a/loro-js/tests/serde-columnar.test.js +++ /dev/null @@ -1,84 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { decodeAnyRleU32, decodeBoolRle, decodeColumnarVec, decodeDeltaOfDeltaI64, decodeDeltaRleI32, decodeDeltaRleIsize, decodeDeltaRleU32, decodeDeltaRleUsize, decodeRleU8, encodeBoolRle, encodeColumnarVec, encodeDeltaOfDeltaI64, encodeDeltaRleI32, encodeDeltaRleIsize, encodeDeltaRleU32, encodeDeltaRleUsize, takeAnyRleU32, takeBoolRle, takeDeltaOfDeltaI64, bytesToHex, } from "../src/codec/index"; -describe("serde-columnar", () => { - test("matches the outer vector format", () => { - const encoded = encodeColumnarVec([ - new TextEncoder().encode("abc"), - Uint8Array.of(1, 2), - ]); - expect(bytesToHex(encoded)).toBe("0203616263020102"); - expect(decodeColumnarVec(encoded)).toEqual([ - new TextEncoder().encode("abc"), - Uint8Array.of(1, 2), - ]); - }); - test("decodes known BoolRle and AnyRle runs", () => { - expect(decodeBoolRle(Uint8Array.of(0, 2, 3, 1))).toEqual([ - true, - true, - false, - false, - false, - true, - ]); - expect(decodeRleU8(Uint8Array.of(6, 5, 4, 3))).toEqual([5, 5, 5, 3, 3]); - expect(decodeAnyRleU32(Uint8Array.of(5, 1, 2, 3))).toEqual([1, 2, 3]); - }); - test("supports prefix decoding from concatenated streams", () => { - const boolBytes = Uint8Array.of(0, 2, 3, 1, 99); - const [bools, boolRest] = takeBoolRle(boolBytes, 6); - expect(bools).toEqual([true, true, false, false, false, true]); - expect(boolRest).toEqual(Uint8Array.of(99)); - const anyBytes = Uint8Array.of(5, 1, 2, 3, 99); - const [values, anyRest] = takeAnyRleU32(anyBytes, 3); - expect(values).toEqual([1, 2, 3]); - expect(anyRest).toEqual(Uint8Array.of(99)); - }); - test("round trips BoolRle", () => { - const values = [false, false, true, true, false, true, true, true]; - expect(decodeBoolRle(encodeBoolRle(values))).toEqual(values); - }); - test("decodes the known DeltaRle run encoding", () => { - expect(decodeDeltaRleU32(Uint8Array.of(2, 20, 6, 2, 4, 4))).toEqual([ - 10, 11, 12, 13, 15, 17, - ]); - }); - test("preserves the canonical literal DeltaRle encoding", () => { - expect(bytesToHex(encodeDeltaRleU32([10, 11, 12, 13, 15, 17]))).toBe("0b140202020404"); - expect(bytesToHex(encodeDeltaRleI32([-2, -1, 0, 5, 3]))).toBe("090302020a03"); - }); - test("round trips the full i32 and u32 delta range", () => { - const unsigned = [0, 4294967295, 0, 4294967295]; - const signed = [-2147483648, 2147483647, -2147483648]; - expect(decodeDeltaRleU32(encodeDeltaRleU32(unsigned))).toEqual(unsigned); - expect(decodeDeltaRleI32(encodeDeltaRleI32(signed))).toEqual(signed); - }); - test("round trips DeltaRle integer variants", () => { - const unsigned = [0, 1, 2, 10, 11, 5]; - const signed = [-2, -1, 0, 5, 3]; - const wideUnsigned = unsigned.map(BigInt); - const wideSigned = signed.map(BigInt); - expect(decodeDeltaRleU32(encodeDeltaRleU32(unsigned))).toEqual(unsigned); - expect(decodeDeltaRleI32(encodeDeltaRleI32(signed))).toEqual(signed); - expect(decodeDeltaRleUsize(encodeDeltaRleUsize(wideUnsigned))).toEqual(wideUnsigned); - expect(decodeDeltaRleIsize(encodeDeltaRleIsize(wideSigned))).toEqual(wideSigned); - }); - test.each([ - { values: [], hex: "0000" }, - { values: [5n], hex: "010a00" }, - { values: [1n, 2n, 3n], hex: "010202a000" }, - ])("matches known DeltaOfDelta bytes $hex", ({ values, hex }) => { - const encoded = encodeDeltaOfDeltaI64(values); - expect(bytesToHex(encoded)).toBe(hex); - expect(decodeDeltaOfDeltaI64(encoded)).toEqual(values); - }); - test("takes a DeltaOfDelta prefix without consuming the next stream", () => { - const encoded = encodeDeltaOfDeltaI64([10n, 12n, 15n]); - const combined = new Uint8Array(encoded.length + 2); - combined.set(encoded); - combined.set([7, 8], encoded.length); - const [values, rest] = takeDeltaOfDeltaI64(combined, 3); - expect(values).toEqual([10n, 12n, 15n]); - expect(rest).toEqual(Uint8Array.of(7, 8)); - }); -}); diff --git a/loro-js/tests/sstable.test.js b/loro-js/tests/sstable.test.js deleted file mode 100644 index 7559679e8..000000000 --- a/loro-js/tests/sstable.test.js +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { LoroDecodeError, decodeSstable, encodeSstable } from "../src/codec/index"; -describe("SSTable", () => { - test("uses zero bytes for an empty KV store", () => { - expect(encodeSstable([])).toEqual(new Uint8Array()); - expect(decodeSstable(new Uint8Array())).toEqual([]); - }); - test.each(["none", "auto", "lz4"])("round trips with %s compression", (compression) => { - const entries = [ - { key: Uint8Array.of(1), value: new Uint8Array() }, - { key: Uint8Array.of(1, 2), value: Uint8Array.of(4, 5) }, - { key: Uint8Array.of(2), value: new Uint8Array(5000).fill(9) }, - ]; - expect(decodeSstable(encodeSstable(entries, { compression }))).toEqual(entries); - }); - test("sorts keys and rejects duplicates", () => { - const encoded = encodeSstable([ - { key: Uint8Array.of(2), value: Uint8Array.of(2) }, - { key: Uint8Array.of(1), value: Uint8Array.of(1) }, - ]); - expect(decodeSstable(encoded).map((entry) => entry.key[0])).toEqual([1, 2]); - expect(() => encodeSstable([ - { key: Uint8Array.of(1), value: new Uint8Array() }, - { key: Uint8Array.of(1), value: new Uint8Array() }, - ])).toThrow("unique"); - }); - test("checks the document-level block checksum", () => { - const encoded = encodeSstable([{ key: Uint8Array.of(1), value: Uint8Array.of(2) }]); - const corrupted = encoded.slice(); - corrupted[6] = corrupted[6] ^ 1; - expect(() => decodeSstable(corrupted)).toThrow(LoroDecodeError); - }); -}); diff --git a/loro-js/tests/state-snapshot.test.js b/loro-js/tests/state-snapshot.test.js deleted file mode 100644 index 3441f1752..000000000 --- a/loro-js/tests/state-snapshot.test.js +++ /dev/null @@ -1,141 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { ContainerType, decodeContainerStateSnapshot, decodeContainerStateWrapper, decodeStateSnapshotStore, encodeContainerStateSnapshot, encodeContainerStateWrapper, encodePositionArena, encodeStateSnapshotStore, unknownContainerType, } from "../src/codec/index"; -const peer = 0x0102030405060708n; -describe("container state snapshot codec", () => { - test("canonicalizes and round trips map state", () => { - const state = { - kind: ContainerType.Map, - values: [ - ["b", { type: "bool", value: true }], - ["a", { type: "i64", value: 1n }], - ], - deletedKeys: ["d", "c"], - peers: [peer], - metadata: [ - { key: "d", peerIndex: 0n, lamport: 4n }, - { key: "b", peerIndex: 0n, lamport: 2n }, - { key: "a", peerIndex: 0n, lamport: 1n }, - { key: "c", peerIndex: 0n, lamport: 3n }, - ], - }; - expect(decodeContainerStateSnapshot(ContainerType.Map, encodeContainerStateSnapshot(state))).toEqual({ - ...state, - values: [...state.values].reverse(), - deletedKeys: [...state.deletedKeys].reverse(), - metadata: [...state.metadata].sort((left, right) => left.key.localeCompare(right.key)), - }); - }); - test("round trips list, text, tree, movable-list, counter and unknown state", () => { - const states = [ - { - kind: ContainerType.List, - values: [{ type: "string", value: "a" }, { type: "null" }], - peers: [peer], - ids: [ - { peerIndex: 0n, counter: 0, lamportSub: 0 }, - { peerIndex: 0n, counter: 1, lamportSub: 1 }, - ], - }, - { - kind: ContainerType.Text, - text: "a😀", - peers: [peer], - spans: [ - { peerIndex: 0n, counter: 0, lamportSub: 0, length: 2 }, - { peerIndex: 0n, counter: 2, lamportSub: 0, length: 0 }, - ], - keys: ["bold"], - marks: [{ keyIndex: 0, value: { type: "bool", value: true }, info: 1 }], - }, - { - kind: ContainerType.Tree, - peers: [peer], - nodes: [ - { - peerIndex: 0n, - counter: 3, - parentIndexPlusTwo: 0n, - lastSetPeerIndex: 0n, - lastSetCounter: 3, - lastSetLamportSub: 0, - fractionalIndexIndex: 0, - }, - ], - positions: [Uint8Array.of(1, 2, 3)], - reserved: new Uint8Array(), - }, - { - kind: ContainerType.MovableList, - values: [{ type: "i64", value: 1n }], - peers: [peer], - items: [ - { - invisibleListItems: 0n, - positionIdEqualsElementId: true, - elementIdEqualsLastSetId: true, - }, - { - invisibleListItems: 0n, - positionIdEqualsElementId: false, - elementIdEqualsLastSetId: false, - }, - ], - listItemIds: [{ peerIndex: 0n, counter: 4, lamportSub: 0 }], - elementIds: [{ peerIndex: 0n, lamport: 4 }], - lastSetIds: [{ peerIndex: 0n, lamport: 5 }], - }, - { kind: ContainerType.Counter, bits: 0x3ff0000000000000n }, - { kind: unknownContainerType(19), payload: Uint8Array.of(1, 2, 3) }, - ]; - for (const state of states) { - expect(decodeContainerStateSnapshot(state.kind, encodeContainerStateSnapshot(state))).toEqual(state); - } - }); - test("round trips empty tree position arena in its non-empty Rust encoding", () => { - const state = { - kind: ContainerType.Tree, - peers: [], - nodes: [], - positions: [], - reserved: new Uint8Array(), - }; - expect(encodePositionArena([], { encodeEmpty: true }).length).toBeGreaterThan(0); - expect(decodeContainerStateSnapshot(ContainerType.Tree, encodeContainerStateSnapshot(state))).toEqual(state); - }); -}); -describe("state wrapper and store codec", () => { - const state = { - kind: ContainerType.Map, - values: [["answer", { type: "i64", value: 42n }]], - deletedKeys: [], - peers: [peer], - metadata: [{ key: "answer", peerIndex: 0n, lamport: 1n }], - }; - const wrapper = { - containerType: ContainerType.Map, - depth: 1n, - parent: undefined, - state, - }; - test("round trips a wrapper", () => { - expect(decodeContainerStateWrapper(encodeContainerStateWrapper(wrapper))).toEqual(wrapper); - }); - test("preserves absent and empty stores", () => { - for (const store of [{ kind: "absent" }, { kind: "empty" }]) { - expect(decodeStateSnapshotStore(encodeStateSnapshotStore(store))).toEqual(store); - } - }); - test("round trips an SSTable store", () => { - const store = { - kind: "sstable", - frontiers: [{ peer, counter: 0 }], - containers: [ - { - id: { kind: "root", name: "map", containerType: ContainerType.Map }, - wrapper, - }, - ], - }; - expect(decodeStateSnapshotStore(encodeStateSnapshotStore(store, { compression: "none" }))).toEqual(store); - }); -}); diff --git a/loro-js/tests/version-transition.test.js b/loro-js/tests/version-transition.test.js deleted file mode 100644 index 4517c6982..000000000 --- a/loro-js/tests/version-transition.test.js +++ /dev/null @@ -1,790 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { LoroDoc } from "../src/runtime/document"; -describe("indexed version transitions", () => { - test("reports tree checkout deletions at their original sibling indices", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const emptyFrontiers = doc.frontiers(); - const tree = doc.getTree("tree"); - const first = tree.createNode(); - const second = tree.createNode(); - doc.commit(); - let batch; - doc.subscribe((event) => { - batch = event; - }); - doc.checkout(emptyFrontiers); - expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ - type: "tree", - diff: [ - { - target: first.id, - action: "delete", - oldParent: undefined, - oldIndex: 0, - }, - { - target: second.id, - action: "delete", - oldParent: undefined, - oldIndex: 1, - }, - ], - }); - }); - test("revives tree metadata events when checkout restores a node", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const tree = doc.getTree("tree"); - const node = tree.createNode(); - node.data.set("value", {}); - doc.commit(); - const liveFrontiers = doc.frontiers(); - tree.delete(node.id); - doc.commit(); - let batch; - doc.subscribe((event) => { - batch = event; - }); - doc.checkout(liveFrontiers); - expect(batch?.events.find(({ target }) => target === node.data.id)?.diff).toEqual({ - type: "map", - updated: { value: {} }, - }); - expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ - type: "tree", - diff: [ - { - target: node.id, - action: "create", - parent: undefined, - index: 0, - fractionalIndex: node.fractionalIndex(), - }, - ], - }); - }); - test("retreats and restores indexed container state", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const list = doc.getList("list"); - const map = doc.getMap("map"); - const tree = doc.getTree("tree"); - const counter = doc.getCounter("counter"); - text.insert(0, "abcdef"); - list.insert(0, "a"); - list.insert(1, "b"); - map.set("value", 1); - map.set("removed", true); - const firstRoot = tree.createNode(); - const deletedRoot = tree.createNode(); - counter.increment(2); - doc.commit(); - const baseFrontiers = doc.frontiers(); - const base = doc.toJSON(); - text.delete(1, 2); - text.insert(2, "XY"); - list.delete(0, 1); - list.insert(1, "c"); - map.set("value", 2); - map.delete("removed"); - const child = tree.createNode(firstRoot.id); - tree.delete(deletedRoot.id); - counter.increment(3); - doc.commit(); - const latest = doc.toJSON(); - for (let iteration = 0; iteration < 3; iteration += 1) { - doc.checkout(baseFrontiers); - expect(doc.toJSON()).toEqual(base); - expect(tree.has(child.id)).toBe(false); - expect(tree.has(deletedRoot.id)).toBe(true); - doc.checkoutToLatest(); - expect(doc.toJSON()).toEqual(latest); - expect(tree.has(child.id)).toBe(true); - expect(tree.isNodeDeleted(deletedRoot.id)).toBe(true); - } - }); - test("switches between concurrent indexed branches", () => { - const seed = new LoroDoc(); - seed.setPeerId(1); - seed.getText("text").insert(0, "base"); - seed.getMap("map").set("base", true); - seed.commit(); - const snapshot = seed.export({ mode: "snapshot" }); - const left = LoroDoc.fromSnapshot(snapshot); - left.setPeerId(2); - left.getText("text").insert(0, "L"); - left.getMap("map").set("left", true); - left.getCounter("counter").increment(2); - left.getTree("tree").createNode(); - left.commit(); - const leftFrontiers = left.frontiers(); - const leftValue = left.toJSON(); - const right = LoroDoc.fromSnapshot(snapshot); - right.setPeerId(3); - right.getText("text").push("R"); - right.getMap("map").set("right", true); - right.getCounter("counter").increment(3); - right.getTree("tree").createNode(); - right.commit(); - const rightFrontiers = right.frontiers(); - const rightValue = right.toJSON(); - left.import(right.export({ mode: "update", from: left.oplogVersion() })); - const merged = left.toJSON(); - left.checkout(leftFrontiers); - expect(left.toJSON()).toEqual(leftValue); - left.checkout(rightFrontiers); - expect(left.toJSON()).toEqual(rightValue); - left.checkoutToLatest(); - expect(left.toJSON()).toEqual(merged); - }); - test("matches a rebuilt fork across conflicting concurrent versions", () => { - const seed = new LoroDoc(); - seed.setPeerId(1); - const text = seed.getText("text"); - text.insert(0, "abc"); - seed.getList("list").push("base"); - seed.getMap("map").set("value", 0); - const firstRoot = seed.getTree("tree").createNode(); - const secondRoot = seed.getTree("tree").createNode(); - seed.getCounter("counter").increment(1); - seed.commit(); - const baseFrontiers = seed.frontiers(); - const left = seed.fork(); - left.setPeerId(2); - left.getText("text").delete(1, 1); - left.getText("text").insert(1, "L"); - left.getList("list").push("left"); - left.getMap("map").set("value", 1); - left.getTree("tree").move(secondRoot.id, firstRoot.id); - left.getCounter("counter").increment(2); - left.commit(); - const leftFrontiers = left.frontiers(); - const right = seed.fork(); - right.setPeerId(3); - right.getText("text").delete(1, 1); - right.getText("text").insert(2, "R"); - right.getList("list").insert(0, "right"); - right.getMap("map").set("value", 2); - right.getTree("tree").delete(secondRoot.id); - right.getCounter("counter").increment(3); - right.commit(); - const rightFrontiers = right.frontiers(); - left.import(right.export({ mode: "update", from: seed.oplogVersion() })); - const mergedFrontiers = left.frontiers(); - const resets = [ - left.getText("text"), - left.getList("list"), - left.getMap("map"), - left.getTree("tree"), - left.getCounter("counter"), - ].map((container) => vi.spyOn(container, "_reset")); - for (const frontiers of [ - leftFrontiers, - baseFrontiers, - rightFrontiers, - mergedFrontiers, - baseFrontiers, - mergedFrontiers, - ]) { - const expected = left.forkAt(frontiers); - left.checkout(frontiers); - expect(left.toJSON()).toEqual(expected.toJSON()); - expect(left.getText("text").toDelta()).toEqual(expected.getText("text").toDelta()); - } - expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); - }); - test("emits a compact replayable diff without resetting indexed containers", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const list = doc.getList("list"); - const map = doc.getMap("map"); - const tree = doc.getTree("tree"); - const counter = doc.getCounter("counter"); - text.insert(0, "a".repeat(16384)); - list.push("a"); - list.push("b"); - map.set("value", 1); - const root = tree.createNode(); - counter.increment(1); - doc.commit(); - const baseFrontiers = doc.frontiers(); - text.delete(8192, 1); - text.insert(8192, "X"); - list.delete(0, 1); - list.push("c"); - map.set("value", 2); - tree.createNode(root.id); - counter.increment(2); - doc.commit(); - const latestMirror = doc.fork(); - let batch; - doc.subscribe((event) => { - batch = event; - }); - const resets = [text, list, map, tree, counter].map((container) => vi.spyOn(container, "_reset")); - doc.checkout(baseFrontiers); - expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); - expect(batch).toBeDefined(); - latestMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(latestMirror.toJSON()).toEqual(doc.toJSON()); - const textDiff = batch.events.find(({ target }) => target === text.id)?.diff; - expect(JSON.stringify(textDiff).length).toBeLessThan(256); - const baseMirror = doc.forkAt(baseFrontiers); - batch = undefined; - doc.checkoutToLatest(); - expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); - expect(batch).toBeDefined(); - baseMirror.applyDiff(batch.events - .filter(({ target }) => target !== tree.id) - .map(({ target, diff }) => [target, diff])); - expect(baseMirror.getText("text").toDelta()).toEqual(text.toDelta()); - expect(baseMirror.getList("list").toJSON()).toEqual(list.toJSON()); - expect(baseMirror.getMap("map").toJSON()).toEqual(map.toJSON()); - expect(baseMirror.getCounter("counter").value).toBe(counter.value); - expect(batch.events.find(({ target }) => target === tree.id)?.diff.type).toBe("tree"); - }); - test("calculates a retreat diff incrementally and restores the current state", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const list = doc.getList("list"); - const map = doc.getMap("map"); - const counter = doc.getCounter("counter"); - text.insert(0, "a".repeat(16384)); - list.push("base"); - map.set("value", 1); - counter.increment(1); - doc.commit(); - const baseFrontiers = doc.frontiers(); - text.delete(8192, 1); - text.insert(8192, "X"); - list.push("latest"); - map.set("value", 2); - counter.increment(2); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const latest = doc.toJSON(); - const mirror = doc.fork(); - const expectedBase = doc.forkAt(baseFrontiers); - const resets = [text, list, map, counter].map((container) => vi.spyOn(container, "_reset")); - const reverse = doc.diff(latestFrontiers, baseFrontiers, false); - expect(resets.every((reset) => reset.mock.calls.length === 0)).toBe(true); - expect(doc.toJSON()).toEqual(latest); - expect(JSON.stringify(reverse).length).toBeLessThan(512); - mirror.applyDiff(reverse); - expect(mirror.getText("text").toDelta()).toEqual(expectedBase.getText("text").toDelta()); - expect(mirror.getList("list").toJSON()).toEqual(expectedBase.getList("list").toJSON()); - expect(mirror.getMap("map").toJSON()).toEqual(expectedBase.getMap("map").toJSON()); - expect(mirror.getCounter("counter").value).toBe(expectedBase.getCounter("counter").value); - }); - test("retreats and reapplies a full-range sequence deletion", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "x".repeat(16384)); - doc.commit(); - const beforeDelete = doc.frontiers(); - const visibleMirror = doc.fork(); - text.delete(0, text.length); - doc.commit(); - const afterDelete = doc.frontiers(); - const deletedMirror = doc.fork(); - doc.checkout(beforeDelete); - expect(text.toString()).toBe("x".repeat(16384)); - doc.checkout(afterDelete); - expect(text.toString()).toBe(""); - let batch; - doc.subscribe((event) => { - batch = event; - }); - doc.checkout(beforeDelete); - expect(batch).toBeDefined(); - deletedMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(deletedMirror.getText("text").toString()).toBe(text.toString()); - batch = undefined; - doc.checkout(afterDelete); - expect(batch).toBeDefined(); - visibleMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(visibleMirror.getText("text").toString()).toBe(text.toString()); - }); - test("retreats and restores a full-range sequence insertion", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const beforeInsert = doc.frontiers(); - const emptyMirror = doc.fork(); - text.insert(0, "x".repeat(16384)); - doc.commit(); - const afterInsert = doc.frontiers(); - const visibleMirror = doc.fork(); - doc.checkout(beforeInsert); - expect(text.toString()).toBe(""); - doc.checkout(afterInsert); - expect(text.toString()).toBe("x".repeat(16384)); - let batch; - doc.subscribe((event) => { - batch = event; - }); - doc.checkout(beforeInsert); - expect(batch).toBeDefined(); - visibleMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(visibleMirror.getText("text").toString()).toBe(text.toString()); - batch = undefined; - doc.checkout(afterInsert); - expect(batch).toBeDefined(); - emptyMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(emptyMirror.getText("text").toString()).toBe(text.toString()); - }); - test("keeps a range hidden while another concurrent deletion is included", () => { - const seed = new LoroDoc(); - seed.setPeerId(1); - seed.getText("text").insert(0, "x".repeat(4096)); - seed.commit(); - const baseFrontiers = seed.frontiers(); - const left = seed.fork(); - left.setPeerId(2); - left.getText("text").delete(0, 4096); - left.commit(); - const leftFrontiers = left.frontiers(); - const right = seed.fork(); - right.setPeerId(3); - right.getText("text").delete(0, 4096); - right.commit(); - const rightFrontiers = right.frontiers(); - left.import(right.export({ mode: "update", from: seed.oplogVersion() })); - expect(left.getText("text").toString()).toBe(""); - left.checkout(rightFrontiers); - expect(left.getText("text").toString()).toBe(""); - left.checkout(leftFrontiers); - expect(left.getText("text").toString()).toBe(""); - left.checkout(baseFrontiers); - expect(left.getText("text").toString()).toBe("x".repeat(4096)); - left.checkoutToLatest(); - expect(left.getText("text").toString()).toBe(""); - }); - test("retreats rich-text marks through per-element style history", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "abcdef"); - text.mark({ start: 1, end: 5 }, "bold", true); - doc.commit(); - const baseFrontiers = doc.frontiers(); - text.unmark({ start: 2, end: 4 }, "bold"); - text.insert(3, "X"); - text.mark({ start: 0, end: 3 }, "bold", true); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const reset = vi.spyOn(text, "_reset"); - for (const frontiers of [ - baseFrontiers, - latestFrontiers, - baseFrontiers, - latestFrontiers, - ]) { - const expected = doc.forkAt(frontiers).getText("text").toDelta(); - doc.checkout(frontiers); - expect(text.toDelta()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("emits replayable range-based rich-text mark transitions", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "a".repeat(16384)); - doc.commit(); - const baseFrontiers = doc.frontiers(); - const baseMirror = doc.fork(); - text.mark({ start: 1024, end: 15360 }, "bold", true); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const latestMirror = doc.fork(); - let batch; - doc.subscribe((event) => { - batch = event; - }); - doc.checkout(baseFrontiers); - expect(batch).toBeDefined(); - latestMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(latestMirror.getText("text").toDelta()).toEqual(text.toDelta()); - expect(JSON.stringify(batch.events[0].diff).length).toBeLessThan(256); - batch = undefined; - doc.checkout(latestFrontiers); - expect(batch).toBeDefined(); - baseMirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(baseMirror.getText("text").toDelta()).toEqual(text.toDelta()); - expect(JSON.stringify(batch.events[0].diff).length).toBeLessThan(256); - }); - test("keeps a subscribed full-range delete and mark transition compact", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - text.insert(0, "x".repeat(16384)); - doc.commit(); - const baseFrontiers = doc.frontiers(); - const mirror = doc.fork(); - text.mark({ start: 0, end: text.length }, "bold", true); - text.delete(0, text.length); - doc.commit(); - const latestFrontiers = doc.frontiers(); - doc.checkout(baseFrontiers); - let batch; - doc.subscribe((event) => { - batch = event; - }); - doc.checkout(latestFrontiers); - expect(batch).toBeDefined(); - expect(JSON.stringify(batch.events[0].diff).length).toBeLessThan(256); - mirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(mirror.getText("text").toDelta()).toEqual(text.toDelta()); - }); - test("retreats movable-list value writes without rebuilding its order", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const list = doc.getMovableList("list"); - list.push("a"); - list.push("b"); - list.push("c"); - doc.commit(); - const baseFrontiers = doc.frontiers(); - list.set(1, "B"); - doc.commit(); - const middleFrontiers = doc.frontiers(); - list.set(1, "latest"); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - baseFrontiers, - latestFrontiers, - middleFrontiers, - baseFrontiers, - latestFrontiers, - ]) { - const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); - doc.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("retreats a movable-list transaction that mixes move, delete, and insert", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const list = doc.getMovableList("list"); - for (const value of ["a", "b", "c", "d"]) - list.push(value); - doc.commit(); - const baseFrontiers = doc.frontiers(); - list.move(1, 3); - list.delete(3, 1); - list.insert(1, "x"); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - baseFrontiers, - latestFrontiers, - baseFrontiers, - latestFrontiers, - ]) { - const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); - doc.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("switches concurrent movable-list moves by replaying branch positions", () => { - const seed = new LoroDoc(); - seed.setPeerId(1); - const seedList = seed.getMovableList("list"); - for (const value of ["a", "b", "c", "d"]) - seedList.push(value); - seed.commit(); - const left = seed.fork(); - left.setPeerId(2); - left.getMovableList("list").move(0, 2); - left.commit(); - const leftFrontiers = left.frontiers(); - const leftValue = left.getMovableList("list").toJSON(); - const right = seed.fork(); - right.setPeerId(3); - right.getMovableList("list").move(0, 1); - right.commit(); - const rightFrontiers = right.frontiers(); - const rightValue = right.getMovableList("list").toJSON(); - left.import(right.export({ mode: "update", from: seed.oplogVersion() })); - const mergedFrontiers = left.frontiers(); - const mergedValue = left.getMovableList("list").toJSON(); - const list = left.getMovableList("list"); - const reset = vi.spyOn(list, "_reset"); - for (const [frontiers, expected] of [ - [leftFrontiers, leftValue], - [rightFrontiers, rightValue], - [leftFrontiers, leftValue], - [mergedFrontiers, mergedValue], - ]) { - left.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - const diffResetCalls = reset.mock.calls.length; - const diff = left.diff(leftFrontiers, rightFrontiers, false); - const mirror = left.forkAt(leftFrontiers); - mirror.applyDiff(diff); - expect(mirror.getMovableList("list").toJSON()).toEqual(rightValue); - expect(reset).toHaveBeenCalledTimes(diffResetCalls); - }); - test("retreats and reapplies a sequential movable-list move suffix", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const list = doc.getMovableList("list"); - for (const value of ["a", "b", "c", "d"]) - list.push(value); - doc.commit(); - const baseFrontiers = doc.frontiers(); - list.move(0, 3); - doc.commit(); - const middleFrontiers = doc.frontiers(); - list.move(1, 0); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - middleFrontiers, - baseFrontiers, - latestFrontiers, - middleFrontiers, - latestFrontiers, - ]) { - const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); - doc.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("retreats a movable-list move suffix through recorded anchors", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const list = doc.getMovableList("list"); - for (const value of ["a", "b", "c", "d"]) - list.push(value); - doc.commit(); - const baseFrontiers = doc.frontiers(); - list.move(0, 2); - doc.commit(); - const middleFrontiers = doc.frontiers(); - list.move(3, 1); - doc.commit(); - const latestFrontiers = doc.frontiers(); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - baseFrontiers, - middleFrontiers, - latestFrontiers, - baseFrontiers, - latestFrontiers, - ]) { - const expected = doc.forkAt(frontiers).getMovableList("list").toJSON(); - doc.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("switches directly between concurrent movable-list move branches", () => { - const base = new LoroDoc(); - base.setPeerId(1); - const baseList = base.getMovableList("list"); - for (const value of ["a", "b", "c", "d"]) - baseList.push(value); - base.commit(); - const left = base.fork(); - left.setPeerId(2); - left.getMovableList("list").move(0, 3); - left.commit(); - const leftFrontiers = left.frontiers(); - const right = base.fork(); - right.setPeerId(3); - right.getMovableList("list").move(3, 0); - right.commit(); - const rightFrontiers = right.frontiers(); - left.import(right.export({ mode: "update" })); - const mergedFrontiers = left.frontiers(); - const list = left.getMovableList("list"); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - leftFrontiers, - rightFrontiers, - mergedFrontiers, - rightFrontiers, - leftFrontiers, - mergedFrontiers, - ]) { - const expected = left.forkAt(frontiers).getMovableList("list").toJSON(); - left.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("switches concurrent movable-list branches with inserted and deleted items", () => { - const base = new LoroDoc(); - base.setPeerId(1); - const baseList = base.getMovableList("list"); - for (const value of ["a", "b", "c", "d"]) - baseList.push(value); - base.commit(); - const left = base.fork(); - left.setPeerId(2); - const leftList = left.getMovableList("list"); - leftList.move(0, 3); - leftList.insert(1, "L"); - left.commit(); - const leftFrontiers = left.frontiers(); - const right = base.fork(); - right.setPeerId(3); - const rightList = right.getMovableList("list"); - rightList.move(3, 0); - rightList.delete(2, 1); - right.commit(); - const rightFrontiers = right.frontiers(); - left.import(right.export({ mode: "update", from: base.oplogVersion() })); - const mergedFrontiers = left.frontiers(); - const list = left.getMovableList("list"); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - leftFrontiers, - rightFrontiers, - mergedFrontiers, - rightFrontiers, - leftFrontiers, - mergedFrontiers, - ]) { - const expected = left.forkAt(frontiers).getMovableList("list").toJSON(); - left.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - }); - test("matches rebuilt movable-list branches across randomized concurrent moves", () => { - let random = 2048475843; - const nextRandom = () => { - random ^= random << 13; - random ^= random >>> 17; - random ^= random << 5; - return random >>> 0; - }; - for (let scenario = 0; scenario < 24; scenario += 1) { - const base = new LoroDoc(); - base.setPeerId(1); - const baseList = base.getMovableList("list"); - for (let index = 0; index < 8; index += 1) - baseList.push(index); - base.commit(); - const left = base.fork(); - left.setPeerId(2); - const leftList = left.getMovableList("list"); - for (let move = 0; move < 4; move += 1) { - const from = nextRandom() % leftList.length; - const to = (from + 1 + (nextRandom() % (leftList.length - 1))) % leftList.length; - leftList.move(from, to); - left.commit(); - } - const leftFrontiers = left.frontiers(); - const right = base.fork(); - right.setPeerId(3); - const rightList = right.getMovableList("list"); - for (let move = 0; move < 4; move += 1) { - const from = nextRandom() % rightList.length; - const to = (from + 1 + (nextRandom() % (rightList.length - 1))) % rightList.length; - rightList.move(from, to); - right.commit(); - } - const rightFrontiers = right.frontiers(); - const importMirror = left.fork(); - let importBatch; - const unsubscribe = left.subscribe((event) => { - importBatch = event; - }); - left.import(right.export({ mode: "update", from: base.oplogVersion() })); - unsubscribe(); - const mergedFrontiers = left.frontiers(); - const list = left.getMovableList("list"); - expect(importBatch).toBeDefined(); - importMirror.applyDiff(importBatch.events.map(({ target, diff }) => [target, diff])); - expect(importMirror.getMovableList("list").toJSON()).toEqual(list.toJSON()); - expect(list.toJSON()).toEqual(left.forkAt(mergedFrontiers).getMovableList("list").toJSON()); - const reset = vi.spyOn(list, "_reset"); - for (const frontiers of [ - leftFrontiers, - rightFrontiers, - mergedFrontiers, - leftFrontiers, - mergedFrontiers, - rightFrontiers, - ]) { - const expected = left.forkAt(frontiers).getMovableList("list").toJSON(); - left.checkout(frontiers); - expect(list.toJSON()).toEqual(expected); - } - expect(reset).not.toHaveBeenCalled(); - const expectedRight = left.forkAt(rightFrontiers).getMovableList("list").toJSON(); - const mirror = left.forkAt(leftFrontiers); - mirror.applyDiff(left.diff(leftFrontiers, rightFrontiers, false)); - expect(mirror.getMovableList("list").toJSON()).toEqual(expectedRight); - expect(reset).not.toHaveBeenCalled(); - } - }); - test("switches a large contiguous text deletion without rebuilding the text", () => { - const doc = new LoroDoc(); - doc.setPeerId(1); - const text = doc.getText("text"); - const value = "x".repeat(16384); - text.insert(0, value); - doc.commit(); - const beforeDelete = doc.frontiers(); - text.delete(0, value.length); - doc.commit(); - expect(text.toString()).toBe(""); - doc.checkout(beforeDelete); - expect(text.toString()).toBe(value); - const mirror = doc.forkAt(beforeDelete); - let batch; - const unsubscribe = doc.subscribe((event) => { - batch = event; - }); - doc.checkoutToLatest(); - unsubscribe(); - expect(text.toString()).toBe(""); - expect(batch).toBeDefined(); - expect(JSON.stringify(batch.events).length).toBeLessThan(256); - mirror.applyDiff(batch.events.map(({ target, diff }) => [target, diff])); - expect(mirror.getText("text").toString()).toBe(""); - }); - test("keeps overlapping concurrent range deletions during branch switches", () => { - const base = new LoroDoc(); - base.setPeerId(1); - base.getText("text").insert(0, "x".repeat(1024)); - base.commit(); - const baseFrontiers = base.frontiers(); - const left = base.fork(); - left.setPeerId(2); - left.getText("text").delete(0, 1024); - left.commit(); - const leftFrontiers = left.frontiers(); - const right = base.fork(); - right.setPeerId(3); - right.getText("text").delete(200, 600); - right.commit(); - const rightFrontiers = right.frontiers(); - left.import(right.export({ mode: "update", from: base.oplogVersion() })); - left.checkout(rightFrontiers); - expect(left.getText("text").length).toBe(424); - left.checkout(baseFrontiers); - expect(left.getText("text").length).toBe(1024); - left.checkout(leftFrontiers); - expect(left.getText("text").length).toBe(0); - }); -}); diff --git a/loro-js/vite.config.js b/loro-js/vite.config.js deleted file mode 100644 index aaef39404..000000000 --- a/loro-js/vite.config.js +++ /dev/null @@ -1,39 +0,0 @@ -import { defineConfig } from "vite-plus"; -export default defineConfig({ - fmt: { - ignorePatterns: ["dist/**", "node_modules/**", "tests/fixtures/**"], - printWidth: 90, - semi: true, - singleQuote: false, - sortPackageJson: false, - trailingComma: "all", - }, - lint: { - ignorePatterns: ["dist/**", "node_modules/**", "tests/fixtures/**"], - options: { - denyWarnings: true, - reportUnusedDisableDirectives: "error", - typeAware: true, - typeCheck: true, - }, - plugins: ["typescript", "oxc", "import", "vitest"], - rules: { - "no-console": "error", - }, - }, - pack: { - clean: true, - dts: true, - entry: ["src/index.ts", "src/codec/index.ts"], - format: "esm", - outDir: "dist", - platform: "neutral", - sourcemap: true, - target: "es2022", - tsconfig: "tsconfig.build.json", - }, - test: { - environment: "node", - include: ["tests/**/*.test.ts"], - }, -}); From b51bb24dd532a1e6272547b3286363e5088e2e16 Mon Sep 17 00:00:00 2001 From: Zixuan Chen Date: Mon, 20 Jul 2026 02:45:02 +0800 Subject: [PATCH 5/5] ci: run loro-js checks --- .github/workflows/loro-js.yml | 31 +++++++++++++++++++++++++++++ loro-js/src/codec/serde-columnar.ts | 14 +++++++++++-- loro-js/src/codec/state-snapshot.ts | 8 ++------ loro-js/src/runtime/event-diff.ts | 9 +++++++-- loro-js/src/runtime/jsonpath.ts | 5 +---- 5 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/loro-js.yml diff --git a/.github/workflows/loro-js.yml b/.github/workflows/loro-js.yml new file mode 100644 index 000000000..b9acc8b2c --- /dev/null +++ b/.github/workflows/loro-js.yml @@ -0,0 +1,31 @@ +name: Test loro-js + +on: + push: + branches: ["main"] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +jobs: + check: + name: Format, lint, typecheck, test, and build + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - name: Setup Node.js 22.x + uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Check loro-js + run: pnpm --filter loro-js check diff --git a/loro-js/src/codec/serde-columnar.ts b/loro-js/src/codec/serde-columnar.ts index b5ba8fd6c..e1fc3f6cb 100644 --- a/loro-js/src/codec/serde-columnar.ts +++ b/loro-js/src/codec/serde-columnar.ts @@ -1,6 +1,13 @@ import { ByteReader, ByteWriter } from "./bytes"; import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; -import { I64_MAX, I64_MIN, U64_MAX, readUleb128, readUlebNumber, writeUleb128 } from "./leb128"; +import { + I64_MAX, + I64_MIN, + U64_MAX, + readUleb128, + readUlebNumber, + writeUleb128, +} from "./leb128"; const U32_MAX = 0xffff_ffff; const I32_MIN = -0x8000_0000; @@ -774,7 +781,10 @@ function readPostcardI128NumberOrBigInt(reader: ByteReader): number | bigint { return encoded % 2 === 0 ? encoded / 2 : -(encoded + 1) / 2; } const value = (encodedBigInt >> 1n) ^ -(encodedBigInt & 1n); - decodeAssert(value >= I128_MIN && value <= I128_MAX, "postcard i128 is out of range"); + decodeAssert( + value >= I128_MIN && value <= I128_MAX, + "postcard i128 is out of range", + ); return value; } multiplier *= 128; diff --git a/loro-js/src/codec/state-snapshot.ts b/loro-js/src/codec/state-snapshot.ts index 34471ed8b..50c0d6dac 100644 --- a/loro-js/src/codec/state-snapshot.ts +++ b/loro-js/src/codec/state-snapshot.ts @@ -898,10 +898,7 @@ function assertDecodedPeerIndex(index: bigint, peerCount: bigint, label: string) } function assertEncodedPeerIndex(index: bigint, peerCount: bigint, label: string): void { - encodeAssert( - index >= 0n && index < peerCount, - `${label} peer index out of range`, - ); + encodeAssert(index >= 0n && index < peerCount, `${label} peer index out of range`); } function validateDecodedText( @@ -975,8 +972,7 @@ function validateEncodedTree(state: TreeStateSnapshot): void { assertEncodedPeerIndex(node.peerIndex, peerCount, "tree node ID"); assertEncodedPeerIndex(node.lastSetPeerIndex, peerCount, "tree last-set ID"); encodeAssert( - node.parentIndexPlusTwo >= 0n && - node.parentIndexPlusTwo <= parentIndexBound, + node.parentIndexPlusTwo >= 0n && node.parentIndexPlusTwo <= parentIndexBound, "tree parent index out of range", ); encodeAssert( diff --git a/loro-js/src/runtime/event-diff.ts b/loro-js/src/runtime/event-diff.ts index 6413f47a2..645f5617c 100644 --- a/loro-js/src/runtime/event-diff.ts +++ b/loro-js/src/runtime/event-diff.ts @@ -61,7 +61,8 @@ export class SequenceEventDiff { const inserted = this.#newNode({ kind: "text", value, - attributes: attributes !== undefined && hasAttributes(attributes) ? attributes : undefined, + attributes: + attributes !== undefined && hasAttributes(attributes) ? attributes : undefined, }); this.#root = mergeNodes(mergeNodes(left, inserted), right); } @@ -242,7 +243,11 @@ function splitPiece(piece: Piece, position: number): [Piece, Piece] { } if (piece.kind === "text") { return [ - { kind: "text", value: piece.value.slice(0, position), attributes: piece.attributes }, + { + kind: "text", + value: piece.value.slice(0, position), + attributes: piece.attributes, + }, { kind: "text", value: piece.value.slice(position), attributes: piece.attributes }, ]; } diff --git a/loro-js/src/runtime/jsonpath.ts b/loro-js/src/runtime/jsonpath.ts index 0db42104a..8d715651c 100644 --- a/loro-js/src/runtime/jsonpath.ts +++ b/loro-js/src/runtime/jsonpath.ts @@ -565,10 +565,7 @@ class FilterParser { #parseComparison(): unknown { const left = this.#parseUnary(); const token = this.#peek(); - if ( - token.kind !== "operator" || - !COMPARISON_OPERATORS.includes(token.value) - ) { + if (token.kind !== "operator" || !COMPARISON_OPERATORS.includes(token.value)) { return left; } this.#index += 1;