diff --git a/.changeset/fuzzy-loros-agree.md b/.changeset/fuzzy-loros-agree.md new file mode 100644 index 000000000..9b67a4b4f --- /dev/null +++ b/.changeset/fuzzy-loros-agree.md @@ -0,0 +1,9 @@ +--- +"loro-js": patch +--- + +Keep semantic no-op detection type-safe so empty arrays, objects, maps, binary +values, and containers cannot be mistaken for one another. Match Rust by +rejecting `isNodeDeleted()` queries for tree nodes that do not exist and by +preserving deleted-tree roots, descendants, positions, and last-move metadata +with the same observable semantics. diff --git a/.github/workflows/loro-js.yml b/.github/workflows/loro-js.yml new file mode 100644 index 000000000..b9acc8b2c --- /dev/null +++ b/.github/workflows/loro-js.yml @@ -0,0 +1,31 @@ +name: Test loro-js + +on: + push: + branches: ["main"] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +jobs: + check: + name: Format, lint, typecheck, test, and build + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v4 + with: + version: 10 + - name: Setup Node.js 22.x + uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Check loro-js + run: pnpm --filter loro-js check diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 528f06694..520aef3a8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -49,3 +49,5 @@ jobs: # run: cargo clippy --all-features -- -Dwarnings - name: Run rust tests run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} deno task test-all + - name: Run Rust and TypeScript interop fuzz smoke + run: pnpm test-loro-js-interop diff --git a/AGENTS.md b/AGENTS.md index 1a9eeb31a..de186026d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,8 @@ Loro is a Rust CRDT workspace with JS/WASM packaging and a MoonBit codec. [skills/loro/SKILL.md](skills/loro/SKILL.md). - Pure TypeScript runtime indexes, complexity contracts, benchmarks, and remaining gaps: [context/loro-js-performance.md](context/loro-js-performance.md). +- Rust/WASM, native Rust, and pure TypeScript differential fuzzing: + [context/loro-js-interop-fuzz.md](context/loro-js-interop-fuzz.md). - Context backlog: [context/CONTEXT-GAPS.md](context/CONTEXT-GAPS.md). ## Commands diff --git a/Cargo.lock b/Cargo.lock index 3a10d3985..1d6930342 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -945,6 +945,7 @@ dependencies = [ "rand 0.8.5", "rayon", "rustc-hash", + "serde", "serde_json", "tabled 0.10.0", "tracing", diff --git a/context/loro-js-interop-fuzz.md b/context/loro-js-interop-fuzz.md new file mode 100644 index 000000000..a2679f429 --- /dev/null +++ b/context/loro-js-interop-fuzz.md @@ -0,0 +1,48 @@ +# Rust / TypeScript interoperability fuzzing + +Verified against code 2026-07-19. + +The executable harness lives under `loro-js/fuzz/`. Its shared scenario schema +is defined in `scenario.mjs`, and `runner.mjs` is the semantic oracle for both +the Rust/WASM package and the pure TypeScript runtime. + +For every command, `runDifferentialScenario` crosses transport blobs between +the two implementations and compares canonical state, history metadata, and +events. `runMalformedImportChecks` mutates Rust- and TypeScript-produced current +update/snapshot blobs and requires matching acceptance or rejection; rejected +imports must not partially change the target. Raw encoded bytes are not compared +because current Loro encodings are wire-compatible but non-canonical. + +`crates/fuzz/src/bin/loro_js_interop_driver.rs` consumes the same JSON schema. +When `LORO_INTEROP_NATIVE_DRIVER` is set, the Node runner checks JS-produced +blobs in native Rust and native-Rust-produced blobs in JS for every permanent +corpus scenario. + +The stable generator avoids semantic no-op and invalid operations whose oplog +policy differs independently of visible CRDT behavior. It also omits rich-text +mark/unmark operations and does not compare events when a snapshot is imported +into a non-empty document. The strict corpus records these known gaps. In +particular, `fuzz/corpus/strict/richtext-anchor.json` demonstrates the remaining +anchor/entity-position mismatch after a marked range receives a later insert. + +Event normalization is payload-only. Subscription callbacks enqueue raw event +batches, which are normalized after the public command returns. Do not query the +live document from a callback: the WASM adapter flushes queued callbacks at the +outer JS method boundary while the pure TypeScript runtime emits during the +internal commit, so callback-time state can differ even when event payloads and +the resulting document state agree. + +Map event recording is operation-aware rather than only comparing the values at +the start and end of a batch. If a key visibly changes and is then restored in +the same local or imported change, Rust reports the final value in `updated`, so +the TypeScript runtime must retain that key. An incoming same-value winner that +never changes the visible value remains an event no-op. The regression scenario +is `fuzz/corpus/local-map-delete-restore-event.json`. + +Run the complete green smoke from the repository root with: + +```sh +pnpm test-loro-js-interop +``` + +See `loro-js/fuzz/README.md` for profiles, replay controls, and corpus promotion. diff --git a/crates/examples/examples/b4_bench.rs b/crates/examples/examples/b4_bench.rs index e7f3d660f..9bd4fda9e 100644 --- a/crates/examples/examples/b4_bench.rs +++ b/crates/examples/examples/b4_bench.rs @@ -69,7 +69,10 @@ fn report() { let d = apply(&actions, 1); d.export(ExportMode::Snapshot).unwrap() }); - println!(" export(+apply,nocache):{:>8.2?} (includes a fresh apply)", t_export_nc); + println!( + " export(+apply,nocache):{:>8.2?} (includes a fresh apply)", + t_export_nc + ); // ---- Snapshot import ---- let mem_before = get_mem_usage(); @@ -89,7 +92,10 @@ fn report() { let v = d.get_deep_value(); std::hint::black_box(v); }); - println!(" import + toJSON: {:>10.2?} (forces full state materialization)", t_import_val); + println!( + " import + toJSON: {:>10.2?} (forces full state materialization)", + t_import_val + ); std::hint::black_box(&imported); // ---- B4 x100 ---- diff --git a/crates/examples/examples/mergeable_notes.rs b/crates/examples/examples/mergeable_notes.rs index 60f1a0ab6..44ffa8148 100644 --- a/crates/examples/examples/mergeable_notes.rs +++ b/crates/examples/examples/mergeable_notes.rs @@ -150,7 +150,10 @@ fn print_report(r: &Report) { println!(" import snapshot: {:?}", r.import_time); println!(" resident after import: {}", r.imported_resident); println!(" get_deep_value: {:?}", r.deep_value_time); - println!(" resident after deep_value: {}", r.after_deep_value_resident); + println!( + " resident after deep_value: {}", + r.after_deep_value_resident + ); } fn main() { diff --git a/crates/fuzz/Cargo.toml b/crates/fuzz/Cargo.toml index 3c128bb72..8502c3eaa 100644 --- a/crates/fuzz/Cargo.toml +++ b/crates/fuzz/Cargo.toml @@ -32,6 +32,7 @@ rayon = "1.10.0" bytes = "1" ensure-cov = { workspace = true } pretty_assertions = "1.4.0" +serde = { workspace = true, features = ["derive"] } loro-alpha-4 = { git = "https://github.com/loro-dev/loro.git", tag = "loro-crdt@1.0.0-alpha.4", package = "loro" } [dev-dependencies] diff --git a/crates/fuzz/src/bin/loro_js_interop_driver.rs b/crates/fuzz/src/bin/loro_js_interop_driver.rs new file mode 100644 index 000000000..1db740880 --- /dev/null +++ b/crates/fuzz/src/bin/loro_js_interop_driver.rs @@ -0,0 +1,600 @@ +use std::{error::Error, io}; + +use loro::{ + ExportMode, Frontiers, LoroDoc, LoroValue, ToJson, TreeID, TreeParentId, VersionVector, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +const SLOT_COUNT: usize = 8; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DriverInput { + scenario: Scenario, + #[serde(default)] + external_blobs: Option>>, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Scenario { + schema_version: u8, + peer_count: usize, + commands: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde( + tag = "kind", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +enum Command { + MapSet { + peer: i64, + key: String, + value: Value, + }, + MapDelete { + peer: i64, + key: String, + }, + ListInsert { + peer: i64, + index: i64, + value: Value, + }, + ListDelete { + peer: i64, + index: i64, + length: i64, + }, + TextInsert { + peer: i64, + index: i64, + text: String, + }, + TextDelete { + peer: i64, + index: i64, + length: i64, + }, + TextMark { + peer: i64, + index: i64, + length: i64, + key: String, + value: Value, + }, + TextUnmark { + peer: i64, + index: i64, + length: i64, + key: String, + }, + MovableInsert { + peer: i64, + index: i64, + value: Value, + }, + MovableDelete { + peer: i64, + index: i64, + length: i64, + }, + MovableSet { + peer: i64, + index: i64, + value: Value, + }, + MovableMove { + peer: i64, + from: i64, + to: i64, + }, + CounterIncrement { + peer: i64, + delta: i64, + }, + TreeCreate { + peer: i64, + parent: Option, + value: Value, + }, + TreeMetaSet { + peer: i64, + node: i64, + key: String, + value: Value, + }, + TreeMove { + peer: i64, + node: i64, + parent: Option, + }, + TreeDelete { + peer: i64, + node: i64, + }, + Commit { + peer: i64, + message: i64, + }, + Enqueue { + source: i64, + target: i64, + slot: i64, + mode: TransportMode, + }, + Deliver { + slot: i64, + copies: i64, + }, + Save { + peer: i64, + checkpoint: i64, + }, + Checkout { + peer: i64, + checkpoint: i64, + }, + Attach { + peer: i64, + }, + Roundtrip { + peer: i64, + mode: TransportMode, + }, +} + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum TransportMode { + Update, + Snapshot, +} + +#[derive(Debug, Clone)] +struct Slot { + blob: Vec, + target: usize, +} + +struct World { + docs: Vec, + slots: Vec>, + checkpoints: Vec>>, + external_blobs: Option>>, + transport_blobs: Vec>, + enqueue_index: usize, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct DriverOutput { + observations: Vec, + transport_blobs: Vec>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Observation { + json: Value, + deep_with_id: Value, + version: Vec<(String, i32)>, + oplog_version: Vec<(String, i32)>, + frontiers: Vec, + oplog_frontiers: Vec, + detached: bool, + shallow: bool, + op_count: usize, + change_count: usize, +} + +fn main() -> Result<(), Box> { + let input: DriverInput = serde_json::from_reader(io::stdin().lock())?; + if input.scenario.schema_version != 1 { + return Err(format!( + "unsupported scenario schema {}", + input.scenario.schema_version + ) + .into()); + } + if input.scenario.peer_count < 2 { + return Err("scenario peerCount must be at least two".into()); + } + + let mut world = World::new(input.scenario.peer_count, input.external_blobs); + for command in &input.scenario.commands { + world.execute(command); + } + let output = DriverOutput { + observations: world.docs.iter().map(observe_doc).collect(), + transport_blobs: world.transport_blobs, + }; + serde_json::to_writer(io::stdout().lock(), &output)?; + Ok(()) +} + +impl World { + fn new(peer_count: usize, external_blobs: Option>>) -> Self { + let docs = (0..peer_count) + .map(|peer| { + let doc = LoroDoc::new(); + doc.set_peer_id(peer as u64 + 1).unwrap(); + doc.set_record_timestamp(false); + doc.set_change_merge_interval(0); + doc.get_map("map"); + doc.get_list("list"); + doc.get_text("text"); + doc.get_movable_list("movable"); + doc.get_counter("counter"); + doc.get_tree("tree").enable_fractional_index(0); + doc + }) + .collect(); + Self { + docs, + slots: vec![None; SLOT_COUNT], + checkpoints: vec![vec![None; SLOT_COUNT]; peer_count], + external_blobs, + transport_blobs: Vec::new(), + enqueue_index: 0, + } + } + + fn execute(&mut self, command: &Command) { + match command { + Command::MapSet { peer, key, value } => { + let _ = self + .doc(*peer) + .get_map("map") + .insert(key, loro_value(value)); + } + Command::MapDelete { peer, key } => { + let map = self.doc(*peer).get_map("map"); + let mut keys = map.keys().map(|key| key.to_string()).collect::>(); + if !keys.is_empty() { + keys.sort(); + let selected = if keys.iter().any(|candidate| candidate == key) { + key + } else { + &keys[0] + }; + let _ = map.delete(selected); + } + } + Command::ListInsert { peer, index, value } => { + let list = self.doc(*peer).get_list("list"); + let _ = list.insert(modulo(*index, list.len() + 1), loro_value(value)); + } + Command::ListDelete { + peer, + index, + length, + } => { + let list = self.doc(*peer).get_list("list"); + if !list.is_empty() { + let index = modulo(*index, list.len()); + let length = 1 + modulo(*length, list.len() - index); + let _ = list.delete(index, length); + } + } + Command::TextInsert { peer, index, text } => { + let text_container = self.doc(*peer).get_text("text"); + let boundaries = utf16_boundaries(&text_container.to_string()); + let _ = + text_container.insert_utf16(boundaries[modulo(*index, boundaries.len())], text); + } + Command::TextDelete { + peer, + index, + length, + } => { + let text = self.doc(*peer).get_text("text"); + let (start, end) = selected_text_range(&text.to_string(), *index, *length); + if start != end { + let _ = text.delete_utf16(start, end - start); + } + } + Command::TextMark { + peer, + index, + length, + key, + value, + } => { + let text = self.doc(*peer).get_text("text"); + let (start, end) = selected_text_range(&text.to_string(), *index, *length); + if start != end { + let _ = text.mark_utf16(start..end, key, loro_value(value)); + } + } + Command::TextUnmark { + peer, + index, + length, + key, + } => { + let text = self.doc(*peer).get_text("text"); + let (start, end) = selected_text_range(&text.to_string(), *index, *length); + if start != end { + let _ = text.unmark_utf16(start..end, key); + } + } + Command::MovableInsert { peer, index, value } => { + let list = self.doc(*peer).get_movable_list("movable"); + let _ = list.insert(modulo(*index, list.len() + 1), loro_value(value)); + } + Command::MovableDelete { + peer, + index, + length, + } => { + let list = self.doc(*peer).get_movable_list("movable"); + if !list.is_empty() { + let index = modulo(*index, list.len()); + let length = 1 + modulo(*length, list.len() - index); + let _ = list.delete(index, length); + } + } + Command::MovableSet { peer, index, value } => { + let list = self.doc(*peer).get_movable_list("movable"); + if !list.is_empty() { + let index = modulo(*index, list.len()); + let current = list + .get(index) + .map(|current| current.get_deep_value().to_json_value()); + if current.as_ref() != Some(value) { + let _ = list.set(index, loro_value(value)); + } + } + } + Command::MovableMove { peer, from, to } => { + let list = self.doc(*peer).get_movable_list("movable"); + if list.len() >= 2 { + let _ = list.mov(modulo(*from, list.len()), modulo(*to, list.len())); + } + } + Command::CounterIncrement { peer, delta } => { + if *delta != 0 { + let _ = self + .doc(*peer) + .get_counter("counter") + .increment(*delta as f64); + } + } + Command::TreeCreate { + peer, + parent, + value, + } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + let parent = parent + .and_then(|raw| nodes.get(modulo(raw, nodes.len())).copied()) + .map(TreeParentId::Node) + .unwrap_or(TreeParentId::Root); + if let Ok(node) = tree.create(parent) { + if let Ok(meta) = tree.get_meta(node) { + let _ = meta.insert("value", loro_value(value)); + } + } + } + Command::TreeMetaSet { + peer, + node, + key, + value, + } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + if let Some(node) = nodes.get(modulo(*node, nodes.len())).copied() { + if let Ok(meta) = tree.get_meta(node) { + let _ = meta.insert(key, loro_value(value)); + } + } + } + Command::TreeMove { peer, node, parent } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + if let Some(target) = nodes.get(modulo(*node, nodes.len())).copied() { + let candidates = nodes + .iter() + .copied() + .filter(|candidate| { + candidate != &target && !is_tree_descendant(&tree, *candidate, target) + }) + .collect::>(); + let selected_parent = parent + .and_then(|raw| candidates.get(modulo(raw, candidates.len())).copied()) + .map(TreeParentId::Node) + .unwrap_or(TreeParentId::Root); + if tree.parent(target).as_ref() != Some(&selected_parent) { + let _ = tree.mov(target, selected_parent); + } + } + } + Command::TreeDelete { peer, node } => { + let tree = self.doc(*peer).get_tree("tree"); + let nodes = live_tree_nodes(&tree); + if let Some(node) = nodes.get(modulo(*node, nodes.len())).copied() { + let _ = tree.delete(node); + } + } + Command::Commit { peer, message } => { + let doc = self.doc(*peer); + doc.set_next_commit_message(&format!("interop-fuzz-{message}")); + doc.commit(); + } + Command::Enqueue { + source, + target, + slot, + mode, + } => self.enqueue(*source, *target, *slot, *mode), + Command::Deliver { slot, copies } => self.deliver(*slot, *copies), + Command::Save { peer, checkpoint } => { + let peer = peer_index(*peer, self.docs.len()); + self.checkpoints[peer][modulo(*checkpoint, SLOT_COUNT)] = + Some(self.docs[peer].state_frontiers()); + } + Command::Checkout { peer, checkpoint } => { + let peer = peer_index(*peer, self.docs.len()); + if let Some(frontiers) = + self.checkpoints[peer][modulo(*checkpoint, SLOT_COUNT)].as_ref() + { + if &self.docs[peer].state_frontiers() != frontiers { + let _ = self.docs[peer].checkout(frontiers); + } + } + } + Command::Attach { peer } => self.doc(*peer).attach(), + Command::Roundtrip { peer, mode } => { + let doc = self.doc(*peer); + let bytes = match mode { + TransportMode::Update => doc.export(ExportMode::all_updates()), + TransportMode::Snapshot => doc.export(ExportMode::Snapshot), + }; + if let Ok(bytes) = bytes { + let imported = LoroDoc::new(); + let _ = imported.import(&bytes); + } + } + } + } + + fn enqueue(&mut self, source: i64, target: i64, slot: i64, mode: TransportMode) { + let source = peer_index(source, self.docs.len()); + let mut target = peer_index(target, self.docs.len()); + if source == target { + target = (target + 1) % self.docs.len(); + } + let own_blob = match mode { + TransportMode::Update => { + self.docs[source].export(ExportMode::updates(&self.docs[target].oplog_vv())) + } + TransportMode::Snapshot => self.docs[source].export(ExportMode::Snapshot), + }; + let Ok(own_blob) = own_blob else { + return; + }; + let blob = self + .external_blobs + .as_ref() + .and_then(|blobs| blobs.get(self.enqueue_index)) + .cloned() + .unwrap_or_else(|| own_blob.clone()); + self.transport_blobs.push(own_blob); + self.enqueue_index += 1; + self.slots[modulo(slot, SLOT_COUNT)] = Some(Slot { blob, target }); + } + + fn deliver(&mut self, slot: i64, copies: i64) { + let Some(slot) = self.slots[modulo(slot, SLOT_COUNT)].clone() else { + return; + }; + let copies = 1 + modulo(copies, 3); + for _ in 0..copies { + let _ = self.docs[slot.target].import(&slot.blob); + } + } + + fn doc(&self, peer: i64) -> &LoroDoc { + &self.docs[peer_index(peer, self.docs.len())] + } +} + +fn observe_doc(doc: &LoroDoc) -> Observation { + Observation { + json: doc.get_deep_value().to_json_value(), + deep_with_id: doc.get_deep_value_with_id().to_json_value(), + version: version_pairs(doc.state_vv()), + oplog_version: version_pairs(doc.oplog_vv()), + frontiers: frontier_strings(doc.state_frontiers()), + oplog_frontiers: frontier_strings(doc.oplog_frontiers()), + detached: doc.is_detached(), + shallow: doc.is_shallow(), + op_count: doc.len_ops(), + change_count: doc.len_changes(), + } +} + +fn version_pairs(version: VersionVector) -> Vec<(String, i32)> { + let mut pairs = version + .iter() + .map(|(peer, counter)| (peer.to_string(), *counter)) + .collect::>(); + pairs.sort_by(|left, right| left.0.cmp(&right.0)); + pairs +} + +fn frontier_strings(frontiers: Frontiers) -> Vec { + let mut values = frontiers + .iter() + .map(|id| format!("{}@{}", id.counter, id.peer)) + .collect::>(); + values.sort(); + values +} + +fn live_tree_nodes(tree: &loro::LoroTree) -> Vec { + let mut nodes = tree + .get_nodes(false) + .into_iter() + .map(|node| node.id) + .collect::>(); + nodes.sort_by_key(ToString::to_string); + nodes +} + +fn is_tree_descendant(tree: &loro::LoroTree, node: TreeID, ancestor: TreeID) -> bool { + let mut parent = tree.parent(node); + while let Some(TreeParentId::Node(parent_id)) = parent { + if parent_id == ancestor { + return true; + } + parent = tree.parent(parent_id); + } + false +} + +fn utf16_boundaries(value: &str) -> Vec { + let mut output = vec![0]; + let mut offset = 0; + for character in value.chars() { + offset += character.len_utf16(); + output.push(offset); + } + output +} + +fn selected_text_range(value: &str, raw_index: i64, raw_length: i64) -> (usize, usize) { + let boundaries = utf16_boundaries(value); + let start_index = modulo(raw_index, boundaries.len()); + let remaining = boundaries.len() - start_index - 1; + if remaining == 0 { + return (boundaries[start_index], boundaries[start_index]); + } + let scalar_length = 1 + modulo(raw_length, remaining); + ( + boundaries[start_index], + boundaries[start_index + scalar_length], + ) +} + +fn loro_value(value: &Value) -> LoroValue { + LoroValue::from(value.clone()) +} + +fn peer_index(value: i64, peer_count: usize) -> usize { + modulo(value, peer_count) +} + +fn modulo(value: i64, length: usize) -> usize { + if length == 0 { + return 0; + } + value.rem_euclid(length as i64) as usize +} diff --git a/crates/fuzz/src/container/map.rs b/crates/fuzz/src/container/map.rs index 84951da5d..b9e22da28 100644 --- a/crates/fuzz/src/container/map.rs +++ b/crates/fuzz/src/container/map.rs @@ -209,7 +209,9 @@ impl Actionable for MapAction { // Type-conflict rejection (`ArgErr`) is an expected outcome — swallow it // here rather than via `unwrap` so unrelated `ArgErr` sources still panic. let result = match kind { - ContainerType::Map => handler.ensure_mergeable_map(key).map(|m| m.to_container()), + ContainerType::Map => { + handler.ensure_mergeable_map(key).map(|m| m.to_container()) + } ContainerType::List => { handler.ensure_mergeable_list(key).map(|l| l.to_container()) } @@ -222,9 +224,9 @@ impl Actionable for MapAction { ContainerType::Tree => { handler.ensure_mergeable_tree(key).map(|t| t.to_container()) } - ContainerType::Counter => { - handler.ensure_mergeable_counter(key).map(|c| c.to_container()) - } + ContainerType::Counter => handler + .ensure_mergeable_counter(key) + .map(|c| c.to_container()), ContainerType::Unknown(_) => return None, }; match result { diff --git a/crates/loro-internal/src/container/richtext/richtext_state.rs b/crates/loro-internal/src/container/richtext/richtext_state.rs index fb8dad91b..db9f02f01 100644 --- a/crates/loro-internal/src/container/richtext/richtext_state.rs +++ b/crates/loro-internal/src/container/richtext/richtext_state.rs @@ -7,6 +7,7 @@ use loro_common::{Counter, IdFull, IdSpan, LoroError, LoroResult, LoroValue, ID} use query::{ByteQuery, ByteQueryT}; use rustc_hash::{FxHashMap, FxHashSet}; use serde::{ser::SerializeStruct, Serialize}; +use smallvec::SmallVec; use std::{ fmt::{Display, Formatter}, ops::{Bound, RangeBounds}, @@ -16,7 +17,6 @@ use std::{ str::Utf8Error, sync::Arc, }; -use smallvec::SmallVec; use tracing::instrument; use crate::{ diff --git a/crates/loro-internal/src/handler.rs b/crates/loro-internal/src/handler.rs index a86e9bb2d..001a36483 100644 --- a/crates/loro-internal/src/handler.rs +++ b/crates/loro-internal/src/handler.rs @@ -1854,12 +1854,9 @@ impl TextHandler { /// /// This method requires auto_commit to be enabled. pub fn splice(&self, pos: usize, len: usize, s: &str, pos_type: PosType) -> LoroResult { - let end = checked_range_end( - pos, - len, - self.len(pos_type), - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, self.len(pos_type), || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; let x = self.slice(pos, end, pos_type)?; self.delete(pos, len, pos_type)?; self.insert(pos, s, pos_type)?; @@ -1968,12 +1965,9 @@ impl TextHandler { } let text_len = self.len(pos_type); - let end = checked_range_end( - pos, - len, - text_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, text_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; self.validate_text_boundary(pos, pos_type)?; self.validate_text_boundary(end, pos_type)?; @@ -2236,12 +2230,9 @@ impl TextHandler { } let text_len = self.len(pos_type); - let end = checked_range_end( - pos, - len, - text_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - ) + let end = checked_range_end(pos, len, text_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + }) .inspect_err(|_| error!("pos={} len={} len={}", pos, len, text_len))?; self.validate_text_boundary(pos, pos_type)?; self.validate_text_boundary(end, pos_type)?; @@ -3245,12 +3236,9 @@ impl ListHandler { match &self.inner { MaybeDetached::Detached(l) => { let mut list = l.lock(); - let end = checked_range_end( - pos, - len, - list.value.len(), - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, list.value.len(), || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; list.value.drain(pos..end); Ok(()) } @@ -3264,12 +3252,9 @@ impl ListHandler { } let list_len = self.len(); - let end = checked_range_end( - pos, - len, - list_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, list_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; let inner = self.inner.try_attached_state()?; let ids: Vec<_> = inner.with_state(|state| { @@ -3919,12 +3904,9 @@ impl MovableListHandler { match &self.inner { MaybeDetached::Detached(d) => { let mut d = d.lock(); - let end = checked_range_end( - pos, - len, - d.value.len(), - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, d.value.len(), || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; d.value.drain(pos..end); Ok(()) } @@ -3939,12 +3921,9 @@ impl MovableListHandler { } let list_len = self.len(); - let end = checked_range_end( - pos, - len, - list_len, - || format!("Position: {}:{}", file!(), line!()).into_boxed_str(), - )?; + let end = checked_range_end(pos, len, list_len, || { + format!("Position: {}:{}", file!(), line!()).into_boxed_str() + })?; let (ids, new_poses) = self.with_state(|state| { let list = state.as_movable_list_state().unwrap(); diff --git a/crates/loro-internal/src/state.rs b/crates/loro-internal/src/state.rs index 0e5009d99..420f1267b 100644 --- a/crates/loro-internal/src/state.rs +++ b/crates/loro-internal/src/state.rs @@ -524,7 +524,26 @@ impl DocState { /// Panic when the diff cannot be merged with the previous diff. /// Caller should call [pre_txn] before calling this to avoid panic. fn record_diff(&mut self, diff: InternalDocDiff) { - if !self.event_recorder.recording_diff || diff.diff.is_empty() { + if !self.event_recorder.recording_diff { + return; + } + + if diff.diff.is_empty() { + let new_version = (*diff.new_version).to_owned(); + let Some(last_diff) = self.event_recorder.diffs.last_mut() else { + // A state-neutral import still advances the document version. Keep + // the next visible event's `from` frontier aligned with DocState. + self.event_recorder.diff_start_version = Some(new_version); + return; + }; + + if last_diff.origin != diff.origin || !last_diff.can_merge(&diff) { + panic!("should call pre_txn before record_diff"); + } + + // The empty transition belongs to the current event batch. It changes + // no container diff, but its version is still the batch's `to` frontier. + last_diff.new_version = Cow::Owned(new_version); return; } diff --git a/crates/loro-internal/src/state/map_state.rs b/crates/loro-internal/src/state/map_state.rs index 947eb6bcc..d82492a3a 100644 --- a/crates/loro-internal/src/state/map_state.rs +++ b/crates/loro-internal/src/state/map_state.rs @@ -29,9 +29,7 @@ fn translate_with_parent( value: LoroValue, ) -> LoroValue { match parent_id { - Some(parent) => { - loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value) - } + Some(parent) => loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value), None => value, } } diff --git a/crates/loro-internal/src/state/richtext_state.rs b/crates/loro-internal/src/state/richtext_state.rs index fac441bde..0060669fa 100644 --- a/crates/loro-internal/src/state/richtext_state.rs +++ b/crates/loro-internal/src/state/richtext_state.rs @@ -12,9 +12,7 @@ use crate::{ list::list_op, richtext::{ config::StyleConfigMap, - richtext_state::{ - DrainInfo, EntityRanges, IterRangeItem, PosType, RichtextStateChunk, - }, + richtext_state::{DrainInfo, EntityRanges, IterRangeItem, PosType, RichtextStateChunk}, AnchorType, RichtextState as InnerState, StyleKey, StyleOp, Styles, }, }, diff --git a/crates/loro-internal/src/state/tree_state.rs b/crates/loro-internal/src/state/tree_state.rs index e89964a2c..a4207c85f 100644 --- a/crates/loro-internal/src/state/tree_state.rs +++ b/crates/loro-internal/src/state/tree_state.rs @@ -281,25 +281,6 @@ impl NodeChildren { } } - fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { - match self { - NodeChildren::Vec(v) => { - if v.len() >= Self::MAX_SIZE_FOR_ARRAY { - self.upgrade(); - return self.push_child_in_order(pos, id); - } - - if let Some(last) = v.last() { - assert!(last.0 < pos); - } - v.push((pos, id)); - } - NodeChildren::BTree(v) => { - v.push_child_in_order(pos, id); - } - } - } - fn len(&self) -> usize { match self { NodeChildren::Vec(v) => v.len(), @@ -366,14 +347,6 @@ mod btree { self.id_to_leaf_index.insert(id, c.leaf); } - pub(super) fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { - let c = self.tree.push(Elem { - pos: Arc::new(pos.clone()), - id, - }); - self.id_to_leaf_index.insert(id, c.leaf); - } - pub(super) fn delete_child(&mut self, id: &TreeID) { if let Some(leaf) = self.id_to_leaf_index.remove(id) { self.tree.remove_leaf(Cursor { leaf, offset: 0 }); @@ -710,19 +683,28 @@ impl TreeState { Ok(()) } - fn _init_push_tree_node_in_order( + fn _init_tree_node_from_snapshot( &mut self, target: TreeID, parent: TreeParentId, last_move_op: IdFull, position: Option, ) -> Result<(), LoroError> { - debug_assert!(!self.trees.contains_key(&target)); + if self.trees.contains_key(&target) { + return Err(LoroError::internal("duplicate tree node id in snapshot")); + } let entry = self.children.entry(parent).or_default(); let node_position = NodePosition::new(position.clone().unwrap_or_default(), last_move_op.idlp()); - debug_assert!(!entry.has_child(&node_position)); - entry.push_child_in_order(node_position, target); + if entry.has_child(&node_position) { + return Err(LoroError::internal( + "duplicate tree node position in snapshot", + )); + } + // Snapshot bytes are external input. Canonical encoders group siblings + // in order, but interoperable encoders may choose another node traversal. + // Insert by NodePosition instead of asserting that input order is sorted. + entry.insert_child(node_position, target); self.trees.insert( target, TreeStateNode { @@ -1726,7 +1708,7 @@ mod snapshot { for (node_id, node) in node_ids.iter().zip(encoded.nodes) { // PERF: we don't need to mov the deleted node, instead we can cache them // If the parent is TreeParentId::Deleted, then all the nodes afterwards are deleted - tree._init_push_tree_node_in_order( + tree._init_tree_node_from_snapshot( *node_id, match node.parent_idx_plus_two { 0 => TreeParentId::Root, @@ -1827,5 +1809,60 @@ mod snapshot { assert!(TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &bytes), ctx).is_err()); } + + #[test] + fn tree_fast_snapshot_accepts_noncanonical_sibling_order() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Tree); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + let low = FractionalIndex::default(); + let high = FractionalIndex::from_bytes(vec![0x81]); + let positions = PositionArena::from_positions([high.as_bytes(), low.as_bytes()]); + let encoded = EncodedTree { + node_ids: vec![ + EncodedTreeNodeId { + peer_idx: 0, + counter: 0, + }, + EncodedTreeNodeId { + peer_idx: 0, + counter: 1, + }, + ], + nodes: vec![ + EncodedTreeNode { + parent_idx_plus_two: 0, + last_set_peer_idx: 0, + last_set_counter: 0, + last_set_lamport_sub_counter: 0, + fractional_index_idx: 0, + }, + EncodedTreeNode { + parent_idx_plus_two: 0, + last_set_peer_idx: 0, + last_set_counter: 1, + last_set_lamport_sub_counter: 0, + fractional_index_idx: 1, + }, + ], + fractional_indexes: positions.encode().into(), + reserved_has_effect_bool_rle: vec![].into(), + }; + let mut bytes = Vec::new(); + leb128::write::unsigned(&mut bytes, 1).unwrap(); + bytes.extend_from_slice(&1_u64.to_le_bytes()); + bytes.extend_from_slice(&serde_columnar::to_vec(&encoded).unwrap()); + + let tree = + TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &bytes), ctx).unwrap(); + let roots = tree.tree_nodes(); + assert_eq!(roots.len(), 2); + assert_eq!(roots[0].id.counter, 1); + assert_eq!(roots[1].id.counter, 0); + } } } diff --git a/crates/loro-internal/src/txn.rs b/crates/loro-internal/src/txn.rs index 817b53cb1..aa600cc0f 100644 --- a/crates/loro-internal/src/txn.rs +++ b/crates/loro-internal/src/txn.rs @@ -10,7 +10,7 @@ use loro_common::{ContainerType, IdLp, IdSpan, LoroResult}; use loro_delta::{array_vec::ArrayVec, DeltaRopeBuilder}; use rle::{HasLength, Mergable, RleVec, Sliceable}; use rustc_hash::FxHashMap; -use smallvec::{smallvec, SmallVec}; +use smallvec::SmallVec; use crate::{ change::{Change, Lamport, Timestamp}, @@ -569,7 +569,12 @@ impl Transaction { let expected = self .doc .upgrade() - .map(|d| d.state.lock().peer.load(std::sync::atomic::Ordering::Relaxed)) + .map(|d| { + d.state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed) + }) .unwrap_or(found); return Err(LoroError::UnmatchedContext { expected, found }); } @@ -770,43 +775,35 @@ fn change_to_diff( }; let mut op_index = 0; - let mut hint_iter = hints.into_iter(); - let mut current_hint = hint_iter.next(); - - while op_index < container_ops.len() { - let Some(hint) = current_hint.take() else { - unreachable!("Missing hint for op"); - }; - - // Collect ops that belong to this hint - let mut ops_for_hint: SmallVec<[Op; 1]> = smallvec![container_ops[op_index].clone()]; - let mut total_len = container_ops[op_index].atom_len(); - - // If hint spans multiple ops, collect them - while total_len < hint.rle_len() { - op_index += 1; - let next_op_len = container_ops[op_index].atom_len(); - let op = if next_op_len + total_len > hint.rle_len() { - let new_len = hint.rle_len() - total_len; - let left = container_ops[op_index].slice(0, new_len); - let right = container_ops[op_index].slice(new_len, next_op_len); - container_ops[op_index] = right; - op_index -= 1; - left + let mut op_offset = 0; + + for hint in hints { + // Ops and event hints use independent RLE merging rules. Walk the op + // stream by atom offset so a single merged op can be split across + // multiple hints (notably adjacent UTF-16 text deletions). + let mut remaining = hint.rle_len(); + let mut ops_for_hint: SmallVec<[Op; 1]> = SmallVec::new(); + while remaining > 0 { + let op = container_ops + .get(op_index) + .unwrap_or_else(|| unreachable!("Missing op for hint")); + let op_len = op.atom_len(); + let available = op_len - op_offset; + let take = available.min(remaining); + let piece = if op_offset == 0 && take == op_len { + op.clone() } else { - container_ops[op_index].clone() + op.slice(op_offset, op_offset + take) }; - - total_len += op.atom_len(); - ops_for_hint.push(op); + ops_for_hint.push(piece); + remaining -= take; + op_offset += take; + if op_offset == op_len { + op_index += 1; + op_offset = 0; + } } - op_index += 1; - assert_eq!(total_len, hint.rle_len(), "Op/hint length mismatch"); - - // Move to next hint - current_hint = hint_iter.next(); - // Generate diff based on hint type match hint { EventHint::Mark { start, end, style } => { @@ -981,6 +978,9 @@ fn change_to_diff( .map(|x| x.content_len() as Lamport) .sum::(); } + + assert_eq!(op_offset, 0, "Missing hint for partial op"); + assert_eq!(op_index, container_ops.len(), "Missing hint for op"); } ans @@ -1015,4 +1015,19 @@ mod tests { )); assert!(!doc.app_state().lock().is_in_txn()); } + + #[cfg(feature = "wasm")] + #[test] + fn txn_events_split_merged_utf16_delete_ops() { + let doc = LoroDoc::new(); + let _subscription = doc.subscribe_root(Arc::new(|_| {})); + let text = doc.get_text("text"); + + text.insert_utf16(0, "a𐐷").unwrap(); + text.delete_utf16(1, 2).unwrap(); + text.delete_utf16(0, 1).unwrap(); + doc.commit_then_renew(); + + assert_eq!(text.to_string(), ""); + } } diff --git a/crates/loro-internal/tests/mergeable_container/convergence.rs b/crates/loro-internal/tests/mergeable_container/convergence.rs index d2ad233f7..db9a55512 100644 --- a/crates/loro-internal/tests/mergeable_container/convergence.rs +++ b/crates/loro-internal/tests/mergeable_container/convergence.rs @@ -279,10 +279,7 @@ fn detached_map_ensure_mergeable_counter_rejects_misuse() { .ensure_mergeable_counter("revision") .expect_err("detached ensure_mergeable_* must error"); assert!( - matches!( - err, - loro_common::LoroError::MisuseDetachedContainer { .. } - ), + matches!(err, loro_common::LoroError::MisuseDetachedContainer { .. }), "detached error must be MisuseDetachedContainer; got {err:?}" ); } diff --git a/crates/loro-internal/tests/mergeable_container/delete.rs b/crates/loro-internal/tests/mergeable_container/delete.rs index a032e7d58..0ca6f2a2a 100644 --- a/crates/loro-internal/tests/mergeable_container/delete.rs +++ b/crates/loro-internal/tests/mergeable_container/delete.rs @@ -261,7 +261,10 @@ fn three_peer_delete_then_recreate_converges() { sync(&a, &c); // C recreates "revision" as a text with a post-delete IdLp. - let c_text = c.get_map("state").ensure_mergeable_text("revision").unwrap(); + let c_text = c + .get_map("state") + .ensure_mergeable_text("revision") + .unwrap(); c_text.insert(0, "after-delete", PosType::Unicode).unwrap(); c.commit_then_renew(); diff --git a/crates/loro-internal/tests/mergeable_container/snapshot.rs b/crates/loro-internal/tests/mergeable_container/snapshot.rs index f45a52288..811cb5738 100644 --- a/crates/loro-internal/tests/mergeable_container/snapshot.rs +++ b/crates/loro-internal/tests/mergeable_container/snapshot.rs @@ -291,9 +291,7 @@ fn shallow_snapshot_preserves_losing_kind_state() { // The Map marker now wins LWW on A and is the visible kind; the Text child becomes the loser // whose state lives only at its deterministic cid. for i in 0..6 { - a.get_map("state") - .insert(&format!("noise_{i}"), i) - .unwrap(); + a.get_map("state").insert(&format!("noise_{i}"), i).unwrap(); a.commit_then_renew(); } let a_map = a.get_map("state").ensure_mergeable_map("k").unwrap(); diff --git a/crates/loro-internal/tests/test.rs b/crates/loro-internal/tests/test.rs index 8cd8809b0..3f45bcd73 100644 --- a/crates/loro-internal/tests/test.rs +++ b/crates/loro-internal/tests/test.rs @@ -125,6 +125,38 @@ fn event_from_checkout() { assert!(ran_cloned.load(std::sync::atomic::Ordering::Relaxed)); } +#[test] +fn local_event_starts_after_net_empty_import() { + let source = LoroDoc::new_auto_commit(); + source.set_peer_id(2).unwrap(); + let source_list = source.get_list("source"); + source_list.insert(0, false).unwrap(); + source_list.delete(0, 1).unwrap(); + source.commit_then_renew(); + let imported_frontiers = source.oplog_frontiers(); + + let target = LoroDoc::new_auto_commit(); + target.set_peer_id(1).unwrap(); + let saw_local = Arc::new(AtomicBool::new(false)); + let saw_local_in_callback = saw_local.clone(); + let expected_from = imported_frontiers.clone(); + let _sub = target.subscribe_root(Arc::new(move |event| { + if event.event_meta.by.is_local() { + assert_eq!(event.event_meta.from, expected_from); + saw_local_in_callback.store(true, std::sync::atomic::Ordering::Relaxed); + } + })); + + target + .import(&source.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!(target.oplog_frontiers(), imported_frontiers); + target.get_list("visible").insert(0, true).unwrap(); + target.commit_then_renew(); + + assert!(saw_local.load(std::sync::atomic::Ordering::Relaxed)); +} + #[test] fn handler_in_event() { let doc = LoroDoc::new_auto_commit(); diff --git a/crates/loro/tests/mergeable_public_api.rs b/crates/loro/tests/mergeable_public_api.rs index 12d87ffbc..f98a4f9e0 100644 --- a/crates/loro/tests/mergeable_public_api.rs +++ b/crates/loro/tests/mergeable_public_api.rs @@ -215,8 +215,14 @@ fn loro_map_ensure_mergeable_tree_through_public_api() { let a = doc(1); let b = doc(2); - let a_tree = a.get_map("state").ensure_mergeable_tree("hierarchy").unwrap(); - let b_tree = b.get_map("state").ensure_mergeable_tree("hierarchy").unwrap(); + let a_tree = a + .get_map("state") + .ensure_mergeable_tree("hierarchy") + .unwrap(); + let b_tree = b + .get_map("state") + .ensure_mergeable_tree("hierarchy") + .unwrap(); assert_eq!( a_tree.id(), b_tree.id(), diff --git a/loro-js/README.md b/loro-js/README.md index bcc8bcdc9..44669425e 100644 --- a/loro-js/README.md +++ b/loro-js/README.md @@ -110,13 +110,21 @@ pnpm exec tsgo -p tsconfig.json --noEmit pnpm lint pnpm build pnpm fixtures:rewrite +pnpm fuzz:interop ``` From the repository root, the Rust-side compatibility suite is: ```sh cargo test -p loro --test loro_js_interop +pnpm test-loro-js-interop ``` +The differential fuzz suite crosses Rust/WASM and pure TypeScript update and +snapshot blobs, compares state/history/events after each command, checks +malformed-import atomicity, and replays its permanent corpus through native +Rust. See [`fuzz/README.md`](./fuzz/README.md) for replay and strict-profile +details. + The format implementation follows the repository reference in [`docs/encoding.md`](https://github.com/loro-dev/loro/blob/main/docs/encoding.md). diff --git a/loro-js/fuzz/.gitignore b/loro-js/fuzz/.gitignore new file mode 100644 index 000000000..d4f588edf --- /dev/null +++ b/loro-js/fuzz/.gitignore @@ -0,0 +1 @@ +artifacts/ diff --git a/loro-js/fuzz/README.md b/loro-js/fuzz/README.md new file mode 100644 index 000000000..f957c332e --- /dev/null +++ b/loro-js/fuzz/README.md @@ -0,0 +1,80 @@ +# Rust / TypeScript interoperability fuzzing + +This suite executes one JSON scenario against the Rust/WASM `loro-crdt` +binding and the pure TypeScript `loro-js` runtime. After every command it +compares canonical document state, deep values with container IDs, text delta, +tree state, version/frontier data, pending/history counts, and normalized event +batches. + +Transport commands deliberately cross the implementations: Rust imports the +TypeScript blob and TypeScript imports the Rust blob. Both update and snapshot +formats are covered, along with duplicate and out-of-order delivery, checkout, +attach, and fresh-document round trips. A separate native Rust driver replays +the checked-in corpus in both directions so the oracle does not depend only on +the WASM binding. Deterministically corrupted update/snapshot inputs also check +matching acceptance or rejection and failure atomicity. + +## Run + +Build the Rust/WASM package once, then run the normal local profile: + +```sh +pnpm release-wasm +pnpm --filter loro-js fuzz:interop +``` + +The CI profile runs 500 traces of at most 60 commands. The native profile runs +the permanent corpus through `crates/fuzz/src/bin/loro_js_interop_driver.rs`. + +```sh +pnpm --filter loro-js fuzz:interop:ci +pnpm --filter loro-js fuzz:interop:native +``` + +Useful environment variables are: + +- `LORO_INTEROP_FUZZ_RUNS` +- `LORO_INTEROP_FUZZ_MAX_COMMANDS` +- `LORO_INTEROP_FUZZ_TIME_MS` +- `LORO_INTEROP_FUZZ_SEED` +- `LORO_INTEROP_FUZZ_PATH` + +Fast-check writes the shrunk scenario to `fuzz/artifacts/` and reports its +seed/path. Replay any scenario directly: + +```sh +node loro-js/fuzz/run-interop-fuzz.mjs --replay /absolute/path/to/failure.json +``` + +Promote useful minimized failures into `fuzz/corpus/`. + +## Profiles and oracle boundaries + +The default `stable` profile targets the interoperability surface currently +claimed by `loro-js`. It chooses valid UTF-16 boundaries and effective +map/tree/movable-list operations so a failure is about CRDT behavior rather +than whether an implementation records an invalid or semantic no-op call. +Encoded byte lengths and layouts are never compared because Loro encoding is +not canonical. + +Equivalent event representations are normalized: adjacent delta fragments are +coalesced, insert/delete operations at one cursor are put in a stable order, +and transaction-local tree create/delete pairs are treated as a net no-op. The +normalizer only reads event payloads; it never queries live document state from +inside a subscription callback, because the WASM wrapper and pure TypeScript +runtime can reach the callback at different points within one public method. +Snapshot imports into a non-empty document still have known event differences, +so the stable profile compares their resulting state but not their event list. + +The `strict` profile adds rich-text mark/unmark generation, the corpus under +`fuzz/corpus/strict/`, and exact snapshot-import event comparison: + +```sh +pnpm --filter loro-js fuzz:interop:strict -- --corpus-only +``` + +It currently reproduces the documented rich-text-anchor limitation: after a +mark, Rust encodes later text insert positions in the anchor-bearing entity +coordinate space while the pure TypeScript runtime uses visible Unicode +positions. Keep the strict corpus failing until that implementation gap is +fixed; it is intentionally not part of the green CI gate. diff --git a/loro-js/fuzz/corpus/checkout-pending-tree.json b/loro-js/fuzz/corpus/checkout-pending-tree.json new file mode 100644 index 000000000..42522e35b --- /dev/null +++ b/loro-js/fuzz/corpus/checkout-pending-tree.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "save", "peer": 0, "checkpoint": 0 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": 1 }, + { "kind": "checkout", "peer": 0, "checkpoint": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/checkout-tree-metadata-revival.json b/loro-js/fuzz/corpus/checkout-tree-metadata-revival.json new file mode 100644 index 000000000..211823528 --- /dev/null +++ b/loro-js/fuzz/corpus/checkout-tree-metadata-revival.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 129, "parent": null, "value": {} }, + { "kind": "save", "peer": 51, "checkpoint": 0 }, + { "kind": "tree-delete", "peer": 33, "node": 0 }, + { "kind": "checkout", "peer": 21, "checkpoint": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/checkout-tree-old-indices.json b/loro-js/fuzz/corpus/checkout-tree-old-indices.json new file mode 100644 index 000000000..0e9ae873c --- /dev/null +++ b/loro-js/fuzz/corpus/checkout-tree-old-indices.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "save", "peer": 0, "checkpoint": 3 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": {} }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": {} }, + { "kind": "checkout", "peer": 0, "checkpoint": 3 } + ] +} diff --git a/loro-js/fuzz/corpus/concurrent-tree-reindex.json b/loro-js/fuzz/corpus/concurrent-tree-reindex.json new file mode 100644 index 000000000..93279720c --- /dev/null +++ b/loro-js/fuzz/corpus/concurrent-tree-reindex.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 15, "parent": null, "value": [] }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "enqueue", "source": 0, "target": 0, "slot": 4, "mode": "update" }, + { "kind": "deliver", "slot": 4, "copies": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/core.json b/loro-js/fuzz/corpus/core.json new file mode 100644 index 000000000..975cf3a13 --- /dev/null +++ b/loro-js/fuzz/corpus/core.json @@ -0,0 +1,33 @@ +{ + "schemaVersion": 1, + "peerCount": 3, + "commands": [ + { "kind": "map-set", "peer": 0, "key": "title", "value": "hello" }, + { "kind": "list-insert", "peer": 0, "index": 0, "value": "A" }, + { "kind": "text-insert", "peer": 0, "index": 0, "text": "a😀文" }, + { "kind": "movable-insert", "peer": 0, "index": 0, "value": "card-a" }, + { "kind": "movable-insert", "peer": 0, "index": 1, "value": "card-b" }, + { "kind": "movable-move", "peer": 0, "from": 0, "to": 1 }, + { "kind": "counter-increment", "peer": 0, "delta": 3 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": "root" }, + { "kind": "tree-create", "peer": 0, "parent": 0, "value": "child" }, + { "kind": "commit", "peer": 0, "message": 1 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "deliver", "slot": 0, "copies": 2 }, + { "kind": "map-set", "peer": 1, "key": "title", "value": "from peer 1" }, + { "kind": "text-insert", "peer": 1, "index": 2, "text": "!" }, + { "kind": "commit", "peer": 1, "message": 2 }, + { "kind": "enqueue", "source": 1, "target": 0, "slot": 1, "mode": "snapshot" }, + { "kind": "deliver", "slot": 1, "copies": 0 }, + { + "kind": "text-mark", + "peer": 0, + "index": 1, + "length": 0, + "key": "bold", + "value": true + }, + { "kind": "roundtrip", "peer": 0, "mode": "update" }, + { "kind": "roundtrip", "peer": 1, "mode": "snapshot" } + ] +} diff --git a/loro-js/fuzz/corpus/deleted-tree-position.json b/loro-js/fuzz/corpus/deleted-tree-position.json new file mode 100644 index 000000000..d0a18ea95 --- /dev/null +++ b/loro-js/fuzz/corpus/deleted-tree-position.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "tree-create", + "peer": 0, + "parent": null, + "value": {} + }, + { + "kind": "tree-create", + "peer": 0, + "parent": null, + "value": false + }, + { + "kind": "tree-delete", + "peer": 0, + "node": 1 + }, + { + "kind": "commit", + "peer": 0, + "message": 0 + } + ] +} diff --git a/loro-js/fuzz/corpus/empty-import-event-frontier.json b/loro-js/fuzz/corpus/empty-import-event-frontier.json new file mode 100644 index 000000000..5da46ab84 --- /dev/null +++ b/loro-js/fuzz/corpus/empty-import-event-frontier.json @@ -0,0 +1,43 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "list-insert", + "peer": 57, + "index": 0, + "value": false + }, + { + "kind": "list-delete", + "peer": 21, + "index": 0, + "length": 0 + }, + { + "kind": "enqueue", + "source": 111, + "target": 0, + "slot": 4, + "mode": "update" + }, + { + "kind": "deliver", + "slot": 4, + "copies": 0 + }, + { + "kind": "movable-insert", + "peer": 0, + "index": 0, + "value": {} + }, + { + "kind": "enqueue", + "source": 0, + "target": 0, + "slot": 0, + "mode": "update" + } + ] +} diff --git a/loro-js/fuzz/corpus/import-tree-replay-noop.json b/loro-js/fuzz/corpus/import-tree-replay-noop.json new file mode 100644 index 000000000..e8c772a03 --- /dev/null +++ b/loro-js/fuzz/corpus/import-tree-replay-noop.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "map-set", + "peer": 1, + "key": "a", + "value": false + }, + { + "kind": "enqueue", + "source": 1, + "target": 0, + "slot": 0, + "mode": "update" + }, + { + "kind": "tree-create", + "peer": 0, + "parent": null, + "value": "" + }, + { + "kind": "deliver", + "slot": 0, + "copies": 0 + } + ] +} diff --git a/loro-js/fuzz/corpus/local-map-delete-restore-event.json b/loro-js/fuzz/corpus/local-map-delete-restore-event.json new file mode 100644 index 000000000..0a8500f60 --- /dev/null +++ b/loro-js/fuzz/corpus/local-map-delete-restore-event.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "map-set", + "peer": 69, + "key": "title", + "value": {} + }, + { + "kind": "attach", + "peer": 73 + }, + { + "kind": "map-delete", + "peer": 3, + "key": "a" + }, + { + "kind": "map-set", + "peer": 13, + "key": "title", + "value": {} + }, + { + "kind": "attach", + "peer": 217 + } + ] +} diff --git a/loro-js/fuzz/corpus/local-tree-nested-create.json b/loro-js/fuzz/corpus/local-tree-nested-create.json new file mode 100644 index 000000000..ead52b8df --- /dev/null +++ b/loro-js/fuzz/corpus/local-tree-nested-create.json @@ -0,0 +1,10 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 45, "parent": null, "value": [] }, + { "kind": "tree-create", "peer": 105, "parent": null, "value": null }, + { "kind": "tree-move", "peer": 3, "node": 0, "parent": 0 }, + { "kind": "commit", "peer": 51, "message": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/local-tree-parent-reindex.json b/loro-js/fuzz/corpus/local-tree-parent-reindex.json new file mode 100644 index 000000000..e42b41c88 --- /dev/null +++ b/loro-js/fuzz/corpus/local-tree-parent-reindex.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "commit", "peer": 0, "message": 0 }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "tree-move", "peer": 0, "node": 0, "parent": 0 }, + { "kind": "enqueue", "source": 0, "target": 0, "slot": 0, "mode": "update" } + ] +} diff --git a/loro-js/fuzz/corpus/out-of-order.json b/loro-js/fuzz/corpus/out-of-order.json new file mode 100644 index 000000000..b5d2c2acb --- /dev/null +++ b/loro-js/fuzz/corpus/out-of-order.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "text-insert", "peer": 0, "index": 0, "text": "first" }, + { "kind": "commit", "peer": 0, "message": 1 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "text-insert", "peer": 0, "index": 5, "text": " second" }, + { "kind": "commit", "peer": 0, "message": 2 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 1, "mode": "update" }, + { "kind": "deliver", "slot": 1, "copies": 0 }, + { "kind": "deliver", "slot": 0, "copies": 1 }, + { "kind": "save", "peer": 1, "checkpoint": 0 }, + { "kind": "list-insert", "peer": 1, "index": 0, "value": "local" }, + { "kind": "commit", "peer": 1, "message": 3 }, + { "kind": "checkout", "peer": 1, "checkpoint": 0 }, + { "kind": "attach", "peer": 1 }, + { "kind": "roundtrip", "peer": 1, "mode": "snapshot" } + ] +} diff --git a/loro-js/fuzz/corpus/retain-only-event.json b/loro-js/fuzz/corpus/retain-only-event.json new file mode 100644 index 000000000..abc1bd3a1 --- /dev/null +++ b/loro-js/fuzz/corpus/retain-only-event.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "list-insert", "peer": 217, "index": 0, "value": 0 }, + { "kind": "attach", "peer": 45 }, + { "kind": "list-insert", "peer": 7, "index": 93, "value": {} }, + { "kind": "list-delete", "peer": 3, "index": 15, "length": 0 }, + { "kind": "attach", "peer": 189 } + ] +} diff --git a/loro-js/fuzz/corpus/snapshot-tree-sibling-order.json b/loro-js/fuzz/corpus/snapshot-tree-sibling-order.json new file mode 100644 index 000000000..f9c907803 --- /dev/null +++ b/loro-js/fuzz/corpus/snapshot-tree-sibling-order.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 1, "parent": null, "value": [] }, + { "kind": "tree-create", "peer": 0, "parent": null, "value": [] }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "deliver", "slot": 0, "copies": 0 }, + { "kind": "roundtrip", "peer": 1, "mode": "snapshot" } + ] +} diff --git a/loro-js/fuzz/corpus/strict/richtext-anchor.json b/loro-js/fuzz/corpus/strict/richtext-anchor.json new file mode 100644 index 000000000..7b185fc98 --- /dev/null +++ b/loro-js/fuzz/corpus/strict/richtext-anchor.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "text-insert", "peer": 0, "index": 0, "text": "a😀文" }, + { + "kind": "text-mark", + "peer": 0, + "index": 1, + "length": 0, + "key": "bold", + "value": true + }, + { "kind": "commit", "peer": 0, "message": 1 }, + { "kind": "enqueue", "source": 0, "target": 1, "slot": 0, "mode": "update" }, + { "kind": "deliver", "slot": 0, "copies": 0 }, + { "kind": "text-insert", "peer": 1, "index": 2, "text": "!" }, + { "kind": "commit", "peer": 1, "message": 2 }, + { "kind": "enqueue", "source": 1, "target": 0, "slot": 1, "mode": "update" }, + { "kind": "deliver", "slot": 1, "copies": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/trailing-retain-event.json b/loro-js/fuzz/corpus/trailing-retain-event.json new file mode 100644 index 000000000..ca2f552d5 --- /dev/null +++ b/loro-js/fuzz/corpus/trailing-retain-event.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "movable-insert", + "peer": 33, + "index": 0, + "value": [] + }, + { + "kind": "attach", + "peer": 189 + }, + { + "kind": "movable-insert", + "peer": 189, + "index": 91, + "value": [] + }, + { + "kind": "movable-move", + "peer": 33, + "from": 153, + "to": 0 + }, + { + "kind": "attach", + "peer": 73 + }, + { + "kind": "text-delete", + "peer": 0, + "index": 0, + "length": 0 + } + ] +} diff --git a/loro-js/fuzz/corpus/transient-tree-reindex.json b/loro-js/fuzz/corpus/transient-tree-reindex.json new file mode 100644 index 000000000..62767e937 --- /dev/null +++ b/loro-js/fuzz/corpus/transient-tree-reindex.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "peerCount": 3, + "commands": [ + { + "kind": "tree-create", + "peer": 1, + "parent": null, + "value": false + }, + { + "kind": "tree-create", + "peer": 1, + "parent": null, + "value": false + }, + { + "kind": "tree-delete", + "peer": 1, + "node": 0 + }, + { + "kind": "enqueue", + "source": 1, + "target": 0, + "slot": 0, + "mode": "update" + } + ] +} diff --git a/loro-js/fuzz/corpus/transient-tree-subtree.json b/loro-js/fuzz/corpus/transient-tree-subtree.json new file mode 100644 index 000000000..e0cc69907 --- /dev/null +++ b/loro-js/fuzz/corpus/transient-tree-subtree.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "tree-create", + "peer": 1, + "parent": null, + "value": [] + }, + { + "kind": "tree-create", + "peer": 1, + "parent": 0, + "value": null + }, + { + "kind": "tree-delete", + "peer": 1, + "node": 0 + }, + { + "kind": "enqueue", + "source": 1, + "target": 0, + "slot": 0, + "mode": "update" + } + ] +} diff --git a/loro-js/fuzz/corpus/transient-tree.json b/loro-js/fuzz/corpus/transient-tree.json new file mode 100644 index 000000000..a2bae246a --- /dev/null +++ b/loro-js/fuzz/corpus/transient-tree.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { "kind": "tree-create", "peer": 1, "parent": null, "value": [] }, + { "kind": "tree-delete", "peer": 1, "node": 0 }, + { "kind": "commit", "peer": 1, "message": 0 } + ] +} diff --git a/loro-js/fuzz/corpus/utf16-delete-hint-split.json b/loro-js/fuzz/corpus/utf16-delete-hint-split.json new file mode 100644 index 000000000..251ffb255 --- /dev/null +++ b/loro-js/fuzz/corpus/utf16-delete-hint-split.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "peerCount": 2, + "commands": [ + { + "kind": "text-insert", + "peer": 1, + "index": 0, + "text": "a𐐷" + }, + { + "kind": "text-delete", + "peer": 1, + "index": 1, + "length": 0 + }, + { + "kind": "text-delete", + "peer": 1, + "index": 0, + "length": 0 + }, + { + "kind": "commit", + "peer": 1, + "message": 0 + } + ] +} diff --git a/loro-js/fuzz/run-interop-fuzz.mjs b/loro-js/fuzz/run-interop-fuzz.mjs new file mode 100644 index 000000000..0f2c9f888 --- /dev/null +++ b/loro-js/fuzz/run-interop-fuzz.mjs @@ -0,0 +1,209 @@ +import { spawnSync } from "node:child_process"; +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + writeFileSync, + writeSync, +} from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import fc from "fast-check"; + +import { + canonicalize, + runDifferentialScenario, + runMalformedImportChecks, + runSingleScenario, +} from "./runner.mjs"; +import { assertScenario, scenarioArbitrary } from "./scenario.mjs"; + +const fuzzRoot = path.dirname(fileURLToPath(import.meta.url)); +const packageRoot = path.resolve(fuzzRoot, ".."); +const workspaceRoot = path.resolve(packageRoot, ".."); +const jsEntry = path.join(packageRoot, "dist/index.js"); +const wasmEntries = [ + path.join(workspaceRoot, "crates/loro-wasm/nodejs/index.js"), + path.join(workspaceRoot, "crates/loro-wasm/nodejs/loro_wasm.js"), +]; +const corpusRoot = path.join(fuzzRoot, "corpus"); +const strictCorpusRoot = path.join(corpusRoot, "strict"); +const artifactsRoot = path.join(fuzzRoot, "artifacts"); + +const options = parseArgs(process.argv.slice(2)); +requireBuild(jsEntry, "pnpm --filter loro-js build"); +const wasmEntry = wasmEntries.find(existsSync); +if (wasmEntry === undefined) { + throw new Error(`missing Rust/WASM package; run pnpm release-wasm first`); +} + +const jsEngine = await import(pathToFileURL(jsEntry).href); +const importedWasmEngine = await import(pathToFileURL(wasmEntry).href); +const wasmEngine = + importedWasmEngine.LoroDoc === undefined + ? importedWasmEngine.default + : importedWasmEngine; +const corpus = readCorpus(options.replay, options.strict); +const malformedCases = runMalformedImportChecks(wasmEngine, jsEngine); +let randomRuns = 0; + +for (const { name, scenario } of corpus) { + runDifferentialScenario(wasmEngine, jsEngine, scenario, { + strict: options.strict, + }); + if (process.env.LORO_INTEROP_NATIVE_DRIVER !== undefined) { + verifyNative(name, scenario, jsEngine, process.env.LORO_INTEROP_NATIVE_DRIVER); + } +} + +if (!options.corpusOnly) { + mkdirSync(artifactsRoot, { recursive: true }); + const journalPath = path.join(artifactsRoot, "active-run.txt"); + const journal = openSync(journalPath, "w"); + let runIndex = 0; + const arbitrary = scenarioArbitrary(options.maxCommands, { strict: options.strict }); + const property = fc.property(arbitrary, (scenario) => { + // A Rust panic can poison a lock before fast-check returns its seed/path. + // Persist the deterministic raw-case position first so a killed or wedged + // process can be replayed with `seed` plus `path=`. + writeSync(journal, `${options.seed} ${runIndex}\n`, 0, "utf8"); + runIndex += 1; + runDifferentialScenario(wasmEngine, jsEngine, scenario, { + strict: options.strict, + }); + }); + process.stdout.write( + `loro interop fuzz started: seed=${options.seed} maxCommands=${options.maxCommands}\n`, + ); + let check; + try { + check = fc.check(property, { + numRuns: options.runs, + seed: options.seed, + path: options.path, + interruptAfterTimeLimit: options.timeLimit, + }); + } finally { + closeSync(journal); + } + randomRuns = check.numRuns; + if (check.failed) { + mkdirSync(artifactsRoot, { recursive: true }); + const scenario = check.counterexample?.[0]; + const artifact = path.join(artifactsRoot, `failure-${Date.now()}.json`); + if (scenario !== undefined) { + writeFileSync(artifact, `${JSON.stringify(scenario, null, 2)}\n`); + } + const details = [ + `Loro interop fuzz failed after ${check.numRuns} runs`, + `seed=${check.seed}`, + `path=${check.counterexamplePath ?? ""}`, + `artifact=${artifact}`, + check.errorInstance?.stack ?? check.error ?? "unknown error", + ].join("\n"); + throw new Error(details); + } +} + +process.stdout.write( + `loro interop fuzz passed: profile=${options.strict ? "strict" : "stable"} corpus=${corpus.length} malformed=${malformedCases} random=${randomRuns}\n`, +); + +function parseArgs(args) { + const ci = args.includes("--ci"); + const corpusOnly = args.includes("--corpus-only"); + const strict = args.includes("--strict"); + const replayIndex = args.indexOf("--replay"); + const replay = replayIndex >= 0 ? args[replayIndex + 1] : undefined; + const envRuns = Number.parseInt(process.env.LORO_INTEROP_FUZZ_RUNS ?? "", 10); + const envCommands = Number.parseInt( + process.env.LORO_INTEROP_FUZZ_MAX_COMMANDS ?? "", + 10, + ); + const envSeed = Number.parseInt(process.env.LORO_INTEROP_FUZZ_SEED ?? "", 10); + const envTime = Number.parseInt(process.env.LORO_INTEROP_FUZZ_TIME_MS ?? "", 10); + return { + ci, + corpusOnly, + strict, + replay, + runs: Number.isSafeInteger(envRuns) ? envRuns : ci ? 500 : 100, + maxCommands: Number.isSafeInteger(envCommands) ? envCommands : ci ? 60 : 40, + seed: Number.isSafeInteger(envSeed) ? envSeed : (Date.now() ^ process.pid) | 0, + path: process.env.LORO_INTEROP_FUZZ_PATH, + timeLimit: Number.isSafeInteger(envTime) ? envTime : ci ? 60_000 : undefined, + }; +} + +function requireBuild(entry, command) { + if (!existsSync(entry)) { + throw new Error(`missing ${entry}; run ${command} first`); + } +} + +function readCorpus(replay, strict) { + const files = + replay === undefined + ? [...jsonFiles(corpusRoot), ...(strict ? jsonFiles(strictCorpusRoot) : [])] + : [path.resolve(replay)]; + return files.map((file) => ({ + name: path.basename(file), + scenario: assertScenario(JSON.parse(readFileSync(file, "utf8"))), + })); +} + +function jsonFiles(root) { + return readdirSync(root) + .filter((name) => name.endsWith(".json")) + .sort() + .map((name) => path.join(root, name)); +} + +function verifyNative(name, scenario, js, driver) { + const jsSelf = runSingleScenario(js, scenario); + const nativeSelf = callNative(driver, { scenario }); + compareNative(`${name}: native self`, nativeSelf.observations, jsSelf.observations); + + const nativeFromJs = callNative(driver, { + scenario, + externalBlobs: jsSelf.transportBlobs, + }); + compareNative( + `${name}: JS -> native Rust`, + nativeFromJs.observations, + jsSelf.observations, + ); + + const jsFromNative = runSingleScenario(js, scenario, nativeSelf.transportBlobs); + compareNative( + `${name}: native Rust -> JS`, + jsFromNative.observations, + nativeSelf.observations, + ); +} + +function callNative(driver, input) { + const result = spawnSync(driver, [], { + input: JSON.stringify(input), + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`native driver failed: ${result.stderr || result.stdout}`); + } + return JSON.parse(result.stdout); +} + +function compareNative(label, left, right) { + const canonicalLeft = canonicalize(left); + const canonicalRight = canonicalize(right); + if (JSON.stringify(canonicalLeft) !== JSON.stringify(canonicalRight)) { + throw new Error( + `${label} mismatch\nnative=${JSON.stringify(canonicalLeft)}\njs=${JSON.stringify(canonicalRight)}`, + ); + } +} diff --git a/loro-js/fuzz/runner.mjs b/loro-js/fuzz/runner.mjs new file mode 100644 index 000000000..fd4b8e97a --- /dev/null +++ b/loro-js/fuzz/runner.mjs @@ -0,0 +1,1123 @@ +import assert from "node:assert/strict"; +import { isDeepStrictEqual } from "node:util"; + +export function canonicalize(value) { + if (value === undefined) return { $undefined: true }; + if (typeof value === "number") { + if (Number.isNaN(value)) return { $number: "NaN" }; + if (value === Number.POSITIVE_INFINITY) return { $number: "+Infinity" }; + if (value === Number.NEGATIVE_INFINITY) return { $number: "-Infinity" }; + if (Object.is(value, -0)) return { $number: "-0" }; + return value; + } + if (typeof value === "bigint") return { $bigint: value.toString() }; + if (value instanceof Uint8Array) { + return { $bytes: Buffer.from(value).toString("base64") }; + } + if (value instanceof Map) { + return { + $map: [...value] + .map(([key, child]) => [String(key), canonicalize(child)]) + .sort(([left], [right]) => left.localeCompare(right)), + }; + } + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value === "string" && value.startsWith("idx:")) { + const marker = value.indexOf(", id:cid:"); + if (marker >= 0) return value.slice(marker + ", id:".length); + } + if (value === null || typeof value !== "object") return value; + + const output = {}; + for (const inputKey of Object.keys(value).sort()) { + const outputKey = inputKey === "fractionalIndex" ? "fractional_index" : inputKey; + const child = value[inputKey]; + if (child !== undefined) output[outputKey] = canonicalize(child); + } + if ("fractional_index" in output && !("parent" in output)) output.parent = null; + return output; +} + +function canonicalVersion(version) { + const json = typeof version?.toJSON === "function" ? version.toJSON() : version; + if (json instanceof Map) { + return [...json] + .map(([peer, counter]) => [String(peer), counter]) + .sort(([left], [right]) => left.localeCompare(right)); + } + return canonicalize(json); +} + +function releaseResource(resource) { + resource?.free?.(); +} + +function releaseResources(resources) { + for (const resource of resources) releaseResource(resource); +} + +function usingResource(resource, action) { + try { + return action(resource); + } finally { + releaseResource(resource); + } +} + +function canonicalVersionAndRelease(version) { + return usingResource(version, canonicalVersion); +} + +function canonicalFrontiers(frontiers) { + return frontiers + .map(({ peer, counter }) => `${counter}@${peer}`) + .sort((left, right) => left.localeCompare(right)); +} + +function normalizeDeepWithId(value, key) { + if (Array.isArray(value)) return value.map((child) => normalizeDeepWithId(child)); + if (value === null || typeof value !== "object") return value; + if (key === "meta" && typeof value.cid === "string" && Object.hasOwn(value, "value")) { + return normalizeDeepWithId(value.value, key); + } + return Object.fromEntries( + Object.entries(value).map(([childKey, child]) => [ + childKey, + normalizeDeepWithId(child, childKey), + ]), + ); +} + +function normalizeEventBatch(batch) { + const normalized = canonicalize(batch); + // `origin` is binding metadata rather than a CRDT diff. The pure TS runtime + // does not currently expose every synthetic Rust origin (for example + // "checkout"), so compare the observable transition itself. + delete normalized.origin; + for (const key of ["from", "to"]) { + if (Array.isArray(normalized[key])) { + normalized[key].sort((left, right) => + `${left.counter}@${left.peer}`.localeCompare(`${right.counter}@${right.peer}`), + ); + } + } + if (Array.isArray(normalized.events)) { + const deletedTreeNodes = collectDeletedTreeNodes(normalized.events); + for (const event of normalized.events) { + if (Array.isArray(event.diff?.diff)) { + event.diff.diff = trimPlainTrailingRetains( + canonicalizeDeltaOrder(coalesceDelta(event.diff.diff)), + ); + } + if (event.diff?.type === "tree" && Array.isArray(event.diff.diff)) { + event.diff.diff = removeTransientTreeActions(event.diff.diff); + } + } + normalized.events = normalized.events.filter( + (event) => + eventHasEffect(event) && !targetsTreeNodeDeletedInBatch(event, deletedTreeNodes), + ); + normalized.events.sort((left, right) => + JSON.stringify([left.target, left.path]).localeCompare( + JSON.stringify([right.target, right.path]), + ), + ); + } + return canonicalize(normalized); +} + +function collectDeletedTreeNodes(events) { + const deleted = new Set(); + for (const event of events) { + if (event.diff?.type !== "tree" || !Array.isArray(event.diff.diff)) continue; + const root = event.path?.[0]; + if (typeof root !== "string") continue; + for (const action of event.diff.diff) { + if (action.action === "delete" && typeof action.target === "string") { + deleted.add(JSON.stringify([root, action.target])); + } + } + } + return deleted; +} + +function canonicalizeDeltaOrder(delta) { + const output = []; + let group = []; + const flush = () => { + if (group.length === 0) return; + const deletes = group.reduce((sum, operation) => sum + (operation.delete ?? 0), 0); + if (deletes > 0) output.push({ delete: deletes }); + output.push( + ...coalesceDelta(group.filter((operation) => operation.delete === undefined)), + ); + group = []; + }; + for (const operation of delta) { + if (operation.retain !== undefined) { + flush(); + output.push(operation); + } else { + group.push(operation); + } + } + flush(); + return coalesceDelta(output); +} + +function removeTransientTreeActions(actions) { + const actionsByTarget = new Map(); + for (const action of actions) { + let targetActions = actionsByTarget.get(action.target); + if (targetActions === undefined) { + targetActions = []; + actionsByTarget.set(action.target, targetActions); + } + targetActions.push(action); + } + const finalParents = new Map(); + const finalDeleted = new Set(); + for (const action of actions) { + if (action.action === "delete") { + finalDeleted.add(action.target); + } else { + finalDeleted.delete(action.target); + finalParents.set(action.target, normalizeTreeParent(action.parent)); + } + } + const endsDeleted = (target, visiting = new Set()) => { + if (finalDeleted.has(target)) return true; + const parent = finalParents.get(target); + if (parent === undefined || parent === null || visiting.has(parent)) return false; + visiting.add(parent); + return endsDeleted(parent, visiting); + }; + const transientTargets = new Set(); + for (const [target, targetActions] of actionsByTarget) { + if (targetActions[0]?.action === "create" && endsDeleted(target)) { + transientTargets.add(target); + } + } + + const positions = new Map(); + const countBefore = (parent, index, target) => { + let count = 0; + for (const [candidate, position] of positions) { + if ( + candidate !== target && + transientTargets.has(candidate) && + position.parent === parent && + position.index < index + ) { + count += 1; + } + } + return count; + }; + const removeAt = (parent, index, target) => { + positions.delete(target); + for (const position of positions.values()) { + if (position.parent === parent && position.index > index) position.index -= 1; + } + }; + const insertAt = (parent, index, target) => { + for (const [candidate, position] of positions) { + if (candidate !== target && position.parent === parent && position.index >= index) { + position.index += 1; + } + } + positions.set(target, { parent, index }); + }; + + const output = []; + for (const action of actions) { + const transient = transientTargets.has(action.target); + const normalized = structuredClone(action); + if (action.action === "create") { + const parent = normalizeTreeParent(action.parent); + if (!transient) { + normalized.index -= countBefore(parent, action.index, action.target); + } + insertAt(parent, action.index, action.target); + } else if (action.action === "delete") { + const oldParent = normalizeTreeParent(action.oldParent); + if (!transient) { + normalized.oldIndex -= countBefore(oldParent, action.oldIndex, action.target); + } + const position = positions.get(action.target) ?? { + parent: oldParent, + index: action.oldIndex, + }; + removeAt(position.parent, position.index, action.target); + } else if (action.action === "move") { + const oldParent = normalizeTreeParent(action.oldParent); + const parent = normalizeTreeParent(action.parent); + if (!transient) { + normalized.oldIndex -= countBefore(oldParent, action.oldIndex, action.target); + } + const previous = positions.get(action.target) ?? { + parent: oldParent, + index: action.oldIndex, + }; + removeAt(previous.parent, previous.index, action.target); + if (!transient) { + normalized.index -= countBefore(parent, action.index, action.target); + } + insertAt(parent, action.index, action.target); + } + if (!transient) output.push(normalized); + } + const hasComposedTarget = output.some((action, index) => + output.slice(index + 1).some((candidate) => candidate.target === action.target), + ); + return coalesceTreeActions( + output, + hasComposedTarget ? positions : undefined, + transientTargets, + ); +} + +function coalesceTreeActions(actions, finalPositions, transientTargets) { + const actionsByTarget = new Map(); + for (const action of actions) { + let targetActions = actionsByTarget.get(action.target); + if (targetActions === undefined) { + targetActions = []; + actionsByTarget.set(action.target, targetActions); + } + targetActions.push(action); + } + const departures = []; + if (finalPositions === undefined) { + for (const [target, targetActions] of actionsByTarget) { + const first = targetActions[0]; + if (first.action === "create") continue; + const last = targetActions.at(-1); + const oldParent = normalizeTreeParent(first.oldParent); + const parent = + last.action === "delete" ? undefined : normalizeTreeParent(last.parent); + if (parent !== oldParent) { + departures.push({ target, parent: oldParent, index: first.oldIndex }); + } + } + } + const finalIndex = (target, parent, fallback) => { + const position = finalPositions?.get(target); + if (position === undefined) { + const removedBefore = departures.filter( + (departure) => + departure.target !== target && + departure.parent === parent && + departure.index < fallback, + ).length; + return fallback - removedBefore; + } + let removedBefore = 0; + for (const [candidate, candidatePosition] of finalPositions) { + if ( + transientTargets.has(candidate) && + candidatePosition.parent === position.parent && + candidatePosition.index < position.index + ) { + removedBefore += 1; + } + } + return position.index - removedBefore; + }; + const output = []; + for (const [target, targetActions] of actionsByTarget) { + const first = targetActions[0]; + const last = targetActions.at(-1); + const existedBefore = first.action !== "create"; + const existsAfter = last.action !== "delete"; + if (!existedBefore && !existsAfter) continue; + if (!existedBefore) { + output.push({ + target, + action: "create", + parent: last.parent, + index: finalIndex(target, normalizeTreeParent(last.parent), last.index), + fractional_index: last.fractional_index, + }); + continue; + } + const oldParent = normalizeTreeParent(first.oldParent); + const oldIndex = first.oldIndex; + if (!existsAfter) { + output.push({ target, action: "delete", oldParent, oldIndex }); + continue; + } + const parent = normalizeTreeParent(last.parent); + const index = finalIndex(target, parent, last.index); + if (parent === oldParent && index === oldIndex) continue; + output.push({ + target, + action: "move", + parent, + index, + fractional_index: last.fractional_index, + oldParent, + oldIndex, + }); + } + return output.sort((left, right) => left.target.localeCompare(right.target)); +} + +function normalizeTreeParent(parent) { + return parent ?? null; +} + +function targetsTreeNodeDeletedInBatch(event, deletedTreeNodes) { + const [root, node] = event.path ?? []; + return ( + typeof root === "string" && + typeof node === "string" && + deletedTreeNodes.has(JSON.stringify([root, node])) + ); +} + +function eventHasEffect(event) { + const diff = event.diff; + if (Array.isArray(diff?.diff)) { + if (diff.type === "list" || diff.type === "text") { + return diff.diff.some( + (operation) => + operation.insert !== undefined || + operation.delete !== undefined || + (operation.attributes !== undefined && + Object.keys(operation.attributes).length > 0), + ); + } + return diff.diff.length > 0; + } + if (diff?.type === "map") return Object.keys(diff.updated ?? {}).length > 0; + if (diff?.type === "counter") return diff.increment !== 0; + return true; +} + +function coalesceDelta(delta) { + const output = []; + for (const operation of delta) { + const previous = output.at(-1); + if ( + previous !== undefined && + Object.hasOwn(previous, "insert") && + Object.hasOwn(operation, "insert") && + isDeepStrictEqual(previous.attributes, operation.attributes) + ) { + if (Array.isArray(previous.insert) && Array.isArray(operation.insert)) { + previous.insert.push(...operation.insert); + continue; + } + if (typeof previous.insert === "string" && typeof operation.insert === "string") { + previous.insert += operation.insert; + continue; + } + } + if ( + previous !== undefined && + typeof previous.delete === "number" && + typeof operation.delete === "number" + ) { + previous.delete += operation.delete; + continue; + } + if ( + previous !== undefined && + typeof previous.retain === "number" && + typeof operation.retain === "number" && + isDeepStrictEqual(previous.attributes, operation.attributes) + ) { + previous.retain += operation.retain; + continue; + } + output.push(structuredClone(operation)); + } + return output; +} + +function trimPlainTrailingRetains(delta) { + while (delta.length > 0) { + const operation = delta.at(-1); + if ( + typeof operation?.retain !== "number" || + (operation.attributes !== undefined && Object.keys(operation.attributes).length > 0) + ) { + break; + } + delta.pop(); + } + return delta; +} + +export function observeDoc(doc) { + const pendingLength = doc.getPendingTxnLength(); + const tree = doc.getTree("tree"); + const nodes = tree.getNodes({ withDeleted: true }); + const text = doc.getText("text"); + try { + const treeNodes = nodes + .map((node) => node.toJSON()) + .sort((left, right) => String(left.id).localeCompare(String(right.id))); + const observation = { + json: doc.toJSON(), + deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), + textDelta: text.toDelta(), + treeNodes, + version: canonicalVersionAndRelease(doc.version()), + frontiers: canonicalFrontiers(doc.frontiers()), + detached: doc.isDetached(), + shallow: doc.isShallow(), + pendingLength, + }; + if (pendingLength === 0) { + observation.oplogVersion = canonicalVersionAndRelease(doc.oplogVersion()); + observation.oplogFrontiers = canonicalFrontiers(doc.oplogFrontiers()); + observation.opCount = doc.opCount(); + observation.changeCount = doc.changeCount(); + } + return canonicalize(observation); + } finally { + releaseResources(nodes); + releaseResource(text); + releaseResource(tree); + } +} + +export function observeDocForNative(doc) { + return canonicalize({ + json: doc.toJSON(), + deepWithId: normalizeDeepWithId(doc.getDeepValueWithID()), + version: canonicalVersionAndRelease(doc.version()), + oplogVersion: canonicalVersionAndRelease(doc.oplogVersion()), + frontiers: canonicalFrontiers(doc.frontiers()), + oplogFrontiers: canonicalFrontiers(doc.oplogFrontiers()), + detached: doc.isDetached(), + shallow: doc.isShallow(), + opCount: doc.opCount(), + changeCount: doc.changeCount(), + }); +} + +function errorCategory(error) { + const message = String(error instanceof Error ? error.message : error).toLowerCase(); + if (message.includes("detached")) return "detached"; + if (message.includes("cycle") || message.includes("descendant")) return "tree-cycle"; + if ( + message.includes("range") || + message.includes("position") || + message.includes("index") + ) { + return "range"; + } + if (message.includes("shallow")) return "shallow"; + if (message.includes("checksum")) return "checksum"; + if (message.includes("dependency") || message.includes("pending")) return "dependency"; + return error?.constructor?.name ?? "Error"; +} + +function capture(action) { + try { + const value = action(); + return { ok: true, value: canonicalize(value) }; + } catch (error) { + return { ok: false, error: errorCategory(error) }; + } +} + +function modulo(value, length) { + if (length <= 0) return 0; + return ((value % length) + length) % length; +} + +function peerIndex(value, peerCount) { + return modulo(value, peerCount); +} + +function utf16Boundaries(value) { + const output = [0]; + for (let index = 0; index < value.length; ) { + const codePoint = value.codePointAt(index); + index += codePoint !== undefined && codePoint > 0xffff ? 2 : 1; + output.push(index); + } + return output; +} + +function selectedTextRange(text, rawIndex, rawLength, allowEmpty) { + const boundaries = utf16Boundaries(text.toString()); + const startIndex = modulo(rawIndex, boundaries.length); + const remaining = boundaries.length - startIndex - 1; + if (remaining <= 0) { + return { start: boundaries[startIndex], end: boundaries[startIndex] }; + } + const scalarLength = allowEmpty + ? modulo(rawLength, remaining + 1) + : 1 + modulo(rawLength, remaining); + return { + start: boundaries[startIndex], + end: boundaries[startIndex + scalarLength], + }; +} + +function liveTreeNodes(tree) { + return tree + .getNodes() + .slice() + .sort((left, right) => String(left.id).localeCompare(String(right.id))); +} + +function isTreeDescendant(node, ancestorId) { + let parent = node.parent(); + while (parent !== undefined) { + if (parent.id === ancestorId) { + releaseResource(parent); + return true; + } + const next = parent.parent(); + releaseResource(parent); + parent = next; + } + return false; +} + +function usingLiveTreeNodes(doc, action) { + return usingResource(doc.getTree("tree"), (tree) => { + const nodes = liveTreeNodes(tree); + try { + return action(tree, nodes); + } finally { + releaseResources(nodes); + } + }); +} + +function applyEdit(doc, command) { + switch (command.kind) { + case "map-set": + return usingResource(doc.getMap("map"), (map) => + map.set(command.key, command.value), + ); + case "map-delete": + return usingResource(doc.getMap("map"), (map) => { + const keys = map.keys(); + if (keys.length === 0) return undefined; + const key = keys.includes(command.key) ? command.key : keys.sort()[0]; + return map.delete(key); + }); + case "list-insert": + return usingResource(doc.getList("list"), (list) => + list.insert(modulo(command.index, list.length + 1), command.value), + ); + case "list-delete": + return usingResource(doc.getList("list"), (list) => { + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + const length = 1 + modulo(command.length, list.length - index); + return list.delete(index, length); + }); + case "text-insert": + return usingResource(doc.getText("text"), (text) => { + const boundaries = utf16Boundaries(text.toString()); + return text.insert( + boundaries[modulo(command.index, boundaries.length)], + command.text, + ); + }); + case "text-delete": + return usingResource(doc.getText("text"), (text) => { + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.delete(start, end - start); + }); + case "text-mark": + return usingResource(doc.getText("text"), (text) => { + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.mark({ start, end }, command.key, command.value); + }); + case "text-unmark": + return usingResource(doc.getText("text"), (text) => { + const { start, end } = selectedTextRange( + text, + command.index, + command.length, + false, + ); + if (start === end) return undefined; + return text.unmark({ start, end }, command.key); + }); + case "movable-insert": + return usingResource(doc.getMovableList("movable"), (list) => + list.insert(modulo(command.index, list.length + 1), command.value), + ); + case "movable-delete": + return usingResource(doc.getMovableList("movable"), (list) => { + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + const length = 1 + modulo(command.length, list.length - index); + return list.delete(index, length); + }); + case "movable-set": + return usingResource(doc.getMovableList("movable"), (list) => { + if (list.length === 0) return undefined; + const index = modulo(command.index, list.length); + if ( + isDeepStrictEqual(canonicalize(list.get(index)), canonicalize(command.value)) + ) { + return undefined; + } + return list.set(index, command.value); + }); + case "movable-move": + return usingResource(doc.getMovableList("movable"), (list) => { + if (list.length < 2) return undefined; + return list.move( + modulo(command.from, list.length), + modulo(command.to, list.length), + ); + }); + case "counter-increment": + return command.delta === 0 + ? undefined + : usingResource(doc.getCounter("counter"), (counter) => + counter.increment(command.delta), + ); + case "tree-create": + return usingLiveTreeNodes(doc, (tree, nodes) => { + const parent = + command.parent === null || nodes.length === 0 + ? undefined + : nodes[modulo(command.parent, nodes.length)].id; + const node = tree.createNode(parent); + try { + usingResource(node.data, (data) => data.set("value", command.value)); + return node.id; + } finally { + releaseResource(node); + } + }); + case "tree-meta-set": + return usingLiveTreeNodes(doc, (_tree, nodes) => { + if (nodes.length === 0) return undefined; + return usingResource(nodes[modulo(command.node, nodes.length)].data, (data) => + data.set(command.key, command.value), + ); + }); + case "tree-move": + return usingLiveTreeNodes(doc, (tree, nodes) => { + if (nodes.length === 0) return undefined; + const node = nodes[modulo(command.node, nodes.length)]; + const candidates = nodes.filter( + (candidate) => + candidate.id !== node.id && !isTreeDescendant(candidate, node.id), + ); + const parent = + command.parent === null || candidates.length === 0 + ? undefined + : candidates[modulo(command.parent, candidates.length)]; + const currentParent = node.parent(); + const currentParentId = currentParent?.id; + releaseResource(currentParent); + if (currentParentId === parent?.id) return undefined; + return tree.move(node.id, parent?.id); + }); + case "tree-delete": + return usingLiveTreeNodes(doc, (tree, nodes) => { + if (nodes.length === 0) return undefined; + return tree.delete(nodes[modulo(command.node, nodes.length)].id); + }); + default: + throw new RangeError(`unsupported edit command ${command.kind}`); + } +} + +function createDoc(engine, peer, initializeRoots = true) { + const doc = new engine.LoroDoc(); + if (peer !== undefined) doc.setPeerId(peer + 1); + doc.setRecordTimestamp(false); + doc.setChangeMergeInterval(0); + if (initializeRoots) { + releaseResource(doc.getMap("map")); + releaseResource(doc.getList("list")); + releaseResource(doc.getText("text")); + releaseResource(doc.getMovableList("movable")); + releaseResource(doc.getCounter("counter")); + usingResource(doc.getTree("tree"), (tree) => tree.enableFractionalIndex(0)); + } + return doc; +} + +function exportFor(doc, target, mode) { + if (mode === "snapshot") return doc.export({ mode: "snapshot" }); + const from = target.oplogVersion(); + try { + return doc.export({ mode: "update", from }); + } finally { + releaseResource(from); + } +} + +function compare(label, left, right) { + try { + assert.deepStrictEqual(left, right); + } catch (cause) { + const error = new Error( + `${label}\nleft=${JSON.stringify(left)}\nright=${JSON.stringify(right)}`, + ); + error.cause = cause; + throw error; + } +} + +class EngineWorld { + constructor(engine, scenario, externalBlobs) { + this.engine = engine; + this.docs = Array.from({ length: scenario.peerCount }, (_, peer) => + createDoc(engine, peer), + ); + this.slots = new Map(); + this.checkpoints = this.docs.map(() => new Map()); + this.externalBlobs = externalBlobs?.map((bytes) => new Uint8Array(bytes)); + this.transportBlobs = []; + this.enqueueIndex = 0; + this.eventQueues = this.docs.map(() => []); + this.subscriptions = this.docs.map((doc, peer) => + doc.subscribe((batch) => { + this.eventQueues[peer].push(batch); + }), + ); + } + + execute(command) { + const doc = this.docs[peerIndex(command.peer ?? 0, this.docs.length)]; + if (command.kind.includes("-") && isEditCommand(command.kind)) { + return capture(() => applyEdit(doc, command)); + } + switch (command.kind) { + case "commit": + return capture(() => doc.commit({ message: `interop-fuzz-${command.message}` })); + case "enqueue": { + const sourceIndex = peerIndex(command.source, this.docs.length); + let targetIndex = peerIndex(command.target, this.docs.length); + if (targetIndex === sourceIndex) + targetIndex = (targetIndex + 1) % this.docs.length; + const ownBlob = exportFor( + this.docs[sourceIndex], + this.docs[targetIndex], + command.mode, + ); + const external = this.externalBlobs?.[this.enqueueIndex]; + const blob = external ?? ownBlob; + this.transportBlobs.push([...ownBlob]); + this.enqueueIndex += 1; + this.slots.set(modulo(command.slot, 8), { + blob, + targetIndex, + mode: command.mode, + }); + return { ok: true, value: "enqueued" }; + } + case "deliver": { + const slot = this.slots.get(modulo(command.slot, 8)); + if (slot === undefined) return { ok: true, value: "empty-slot" }; + const copies = 1 + modulo(command.copies, 3); + const statuses = []; + for (let copy = 0; copy < copies; copy += 1) { + statuses.push(canonicalize(this.docs[slot.targetIndex].import(slot.blob))); + } + return { ok: true, value: statuses }; + } + case "save": + this.checkpoints[peerIndex(command.peer, this.docs.length)].set( + modulo(command.checkpoint, 8), + doc.frontiers(), + ); + return { ok: true, value: undefined }; + case "checkout": { + const frontiers = this.checkpoints[peerIndex(command.peer, this.docs.length)].get( + modulo(command.checkpoint, 8), + ); + if (frontiers === undefined) return { ok: true, value: "missing-checkpoint" }; + if ( + isDeepStrictEqual( + canonicalFrontiers(frontiers), + canonicalFrontiers(doc.frontiers()), + ) + ) { + return { ok: true, value: "current-checkpoint" }; + } + return capture(() => doc.checkout(frontiers)); + } + case "attach": + return capture(() => doc.attach()); + case "roundtrip": + return this.roundtrip(doc, command.mode); + default: + throw new RangeError(`unsupported command ${command.kind}`); + } + } + + roundtrip(doc, mode) { + return capture(() => { + const bytes = + mode === "snapshot" + ? doc.export({ mode: "snapshot" }) + : doc.export({ mode: "update" }); + const imported = createDoc(this.engine, undefined, false); + try { + imported.import(bytes); + return observeDoc(imported); + } finally { + releaseResource(imported); + } + }); + } + + finalNativeObservations() { + return this.docs.map(observeDocForNative); + } + + drainEvents() { + this.engine.callPendingEvents?.(); + return this.eventQueues.map((queue) => + queue + .splice(0) + .map(normalizeEventBatch) + .filter((batch) => batch.events.length > 0), + ); + } + + dispose() { + for (const subscription of this.subscriptions) { + if (typeof subscription === "function") subscription(); + else releaseResource(subscription); + } + releaseResources(this.docs); + } +} + +function isEditCommand(kind) { + return ( + kind.startsWith("map-") || + kind.startsWith("list-") || + kind.startsWith("text-") || + kind.startsWith("movable-") || + kind.startsWith("counter-") || + kind.startsWith("tree-") + ); +} + +export function runSingleScenario(engine, scenario, externalBlobs) { + const world = new EngineWorld(engine, scenario, externalBlobs); + try { + const results = []; + for (const command of scenario.commands) results.push(world.execute(command)); + return { + observations: world.finalNativeObservations(), + transportBlobs: world.transportBlobs, + results: canonicalize(results), + }; + } finally { + world.dispose(); + } +} + +export function runMalformedImportChecks(wasmEngine, jsEngine) { + const engines = [ + ["wasm", wasmEngine], + ["js", jsEngine], + ]; + let cases = 0; + for (const [producerName, producer] of engines) { + const source = createDoc(producer, 0); + try { + usingResource(source.getMap("map"), (map) => + map.set("payload", { nested: [1, true, "😀"] }), + ); + usingResource(source.getText("text"), (text) => text.insert(0, "malformed 😀 文")); + usingResource(source.getList("list"), (list) => list.push("item")); + source.commit({ message: "malformed-seed" }); + for (const mode of ["update", "snapshot"]) { + const blob = source.export({ mode }); + for (const [mutation, bytes] of malformedVariants(blob)) { + const outcomes = engines.map(([consumerName, engine]) => [ + consumerName, + malformedOutcome(engine, bytes), + ]); + compare( + `malformed import mismatch: producer=${producerName} mode=${mode} mutation=${mutation}`, + outcomes[0][1], + outcomes[1][1], + ); + cases += 1; + } + } + } finally { + releaseResource(source); + } + } + return cases; +} + +function malformedVariants(blob) { + const half = Math.max(1, Math.floor(blob.length / 2)); + return [ + ["empty", new Uint8Array()], + ["prefix-1", blob.slice(0, 1)], + ["prefix-half", blob.slice(0, half)], + ["truncate-last", blob.slice(0, -1)], + ["append-garbage", Uint8Array.from([...blob, 0xff])], + ["flip-header", flippedByte(blob, 0)], + ["flip-middle", flippedByte(blob, Math.floor(blob.length / 2))], + ["flip-tail", flippedByte(blob, blob.length - 1)], + ]; +} + +function flippedByte(blob, index) { + const output = blob.slice(); + output[index] ^= 0xff; + return output; +} + +function malformedOutcome(engine, bytes) { + const target = createDoc(engine, 97); + try { + usingResource(target.getMap("map"), (map) => map.set("sentinel", "unchanged")); + usingResource(target.getText("text"), (text) => text.insert(0, "stable")); + target.commit({ message: "malformed-target" }); + const before = observeDoc(target); + try { + target.import(bytes); + return { accepted: true, observation: observeDoc(target) }; + } catch { + return { + accepted: false, + atomic: isDeepStrictEqual(before, observeDoc(target)), + }; + } + } finally { + releaseResource(target); + } +} + +export function runDifferentialScenario( + wasmEngine, + jsEngine, + scenario, + { strict = false } = {}, +) { + const wasm = new EngineWorld(wasmEngine, scenario); + const js = new EngineWorld(jsEngine, scenario); + + try { + for (let index = 0; index < scenario.commands.length; index += 1) { + const command = scenario.commands[index]; + let wasmResult; + let jsResult; + let compareEvents = true; + if (command.kind === "deliver") { + const wasmSlot = wasm.slots.get(modulo(command.slot, 8)); + const jsSlot = js.slots.get(modulo(command.slot, 8)); + if (wasmSlot === undefined || jsSlot === undefined) { + wasmResult = { ok: true, value: "empty-slot" }; + jsResult = { ok: true, value: "empty-slot" }; + } else { + compareEvents = strict || wasmSlot.mode !== "snapshot"; + const copies = 1 + modulo(command.copies, 3); + const wasmStatuses = []; + const jsStatuses = []; + for (let copy = 0; copy < copies; copy += 1) { + wasmStatuses.push( + canonicalize(wasm.docs[wasmSlot.targetIndex].import(jsSlot.blob)), + ); + jsStatuses.push( + canonicalize(js.docs[jsSlot.targetIndex].import(wasmSlot.blob)), + ); + } + wasmResult = { ok: true, value: wasmStatuses }; + jsResult = { ok: true, value: jsStatuses }; + } + } else if (command.kind === "roundtrip") { + const peer = peerIndex(command.peer, wasm.docs.length); + wasmResult = crossRoundtrip( + wasmEngine, + jsEngine, + wasm.docs[peer], + js.docs[peer], + command.mode, + ); + jsResult = wasmResult; + } else { + wasmResult = wasm.execute(command); + jsResult = js.execute(command); + } + + compare( + `command result mismatch at ${index}: ${JSON.stringify(command)}`, + wasmResult, + jsResult, + ); + const wasmEvents = wasm.drainEvents(); + const jsEvents = js.drainEvents(); + if (compareEvents) { + compare( + `event mismatch at command ${index}: ${JSON.stringify(command)}`, + wasmEvents, + jsEvents, + ); + } + for (let peer = 0; peer < scenario.peerCount; peer += 1) { + compare( + `document mismatch at command ${index}, peer ${peer}: ${JSON.stringify(command)}`, + observeDoc(wasm.docs[peer]), + observeDoc(js.docs[peer]), + ); + } + } + } finally { + wasm.dispose(); + js.dispose(); + } +} + +function crossRoundtrip(wasmEngine, jsEngine, wasmDoc, jsDoc, mode) { + return capture(() => { + const wasmBytes = + mode === "snapshot" + ? wasmDoc.export({ mode: "snapshot" }) + : wasmDoc.export({ mode: "update" }); + const jsBytes = + mode === "snapshot" + ? jsDoc.export({ mode: "snapshot" }) + : jsDoc.export({ mode: "update" }); + const variants = [ + [wasmEngine, wasmBytes], + [jsEngine, wasmBytes], + [wasmEngine, jsBytes], + [jsEngine, jsBytes], + ].map(([engine, bytes]) => { + const doc = createDoc(engine, undefined, false); + try { + doc.import(bytes); + return observeDoc(doc); + } finally { + releaseResource(doc); + } + }); + for (let index = 1; index < variants.length; index += 1) { + compare(`roundtrip variant ${index} mismatch`, variants[0], variants[index]); + } + return variants[0]; + }); +} diff --git a/loro-js/fuzz/scenario.mjs b/loro-js/fuzz/scenario.mjs new file mode 100644 index 000000000..9c97a71a8 --- /dev/null +++ b/loro-js/fuzz/scenario.mjs @@ -0,0 +1,157 @@ +import fc from "fast-check"; + +const alphabet = ["a", "b", "c", " ", "\n", "\0", "é", "文", "א", "😀", "𐐷", "\u0301"]; + +const smallString = (options = {}) => + fc + .array(fc.constantFrom(...alphabet), { + minLength: options.minLength ?? 0, + maxLength: options.maxLength ?? 8, + }) + .map((parts) => parts.join("")); + +const key = fc.constantFrom("a", "b", "title", "value", "emoji😀", "empty"); +const scalar = fc.oneof( + fc.constant(null), + fc.boolean(), + fc.integer({ min: -1_000, max: 1_000 }), + smallString({ maxLength: 6 }), +); +const value = fc.oneof( + scalar, + fc.array(scalar, { maxLength: 4 }), + fc.dictionary(key, scalar, { maxKeys: 4 }), +); +const byte = fc.integer({ min: 0, max: 255 }); +const peer = byte; +const slot = fc.integer({ min: 0, max: 7 }); + +const commandArbitraries = [ + fc.record({ kind: fc.constant("map-set"), peer, key, value }), + fc.record({ kind: fc.constant("map-delete"), peer, key }), + fc.record({ kind: fc.constant("list-insert"), peer, index: byte, value }), + fc.record({ + kind: fc.constant("list-delete"), + peer, + index: byte, + length: byte, + }), + fc.record({ + kind: fc.constant("text-insert"), + peer, + index: byte, + text: smallString({ minLength: 1, maxLength: 8 }), + }), + fc.record({ + kind: fc.constant("text-delete"), + peer, + index: byte, + length: byte, + }), + fc.record({ + kind: fc.constant("text-mark"), + peer, + index: byte, + length: byte, + key: fc.constantFrom("bold", "link", "comment"), + value: scalar, + }), + fc.record({ + kind: fc.constant("text-unmark"), + peer, + index: byte, + length: byte, + key: fc.constantFrom("bold", "link", "comment"), + }), + fc.record({ kind: fc.constant("movable-insert"), peer, index: byte, value }), + fc.record({ + kind: fc.constant("movable-delete"), + peer, + index: byte, + length: byte, + }), + fc.record({ kind: fc.constant("movable-set"), peer, index: byte, value }), + fc.record({ kind: fc.constant("movable-move"), peer, from: byte, to: byte }), + fc.record({ + kind: fc.constant("counter-increment"), + peer, + delta: fc.integer({ min: -20, max: 20 }), + }), + fc.record({ + kind: fc.constant("tree-create"), + peer, + parent: fc.option(byte, { nil: null }), + value, + }), + fc.record({ + kind: fc.constant("tree-meta-set"), + peer, + node: byte, + key, + value, + }), + fc.record({ + kind: fc.constant("tree-move"), + peer, + node: byte, + parent: fc.option(byte, { nil: null }), + }), + fc.record({ kind: fc.constant("tree-delete"), peer, node: byte }), + fc.record({ kind: fc.constant("commit"), peer, message: byte }), + fc.record({ + kind: fc.constant("enqueue"), + source: peer, + target: peer, + slot, + mode: fc.constantFrom("update", "snapshot"), + }), + fc.record({ kind: fc.constant("deliver"), slot, copies: byte }), + fc.record({ kind: fc.constant("save"), peer, checkpoint: slot }), + fc.record({ kind: fc.constant("checkout"), peer, checkpoint: slot }), + fc.record({ kind: fc.constant("attach"), peer }), + fc.record({ + kind: fc.constant("roundtrip"), + peer, + mode: fc.constantFrom("update", "snapshot"), + }), +]; + +// Rich-text anchors are represented differently by the pure TypeScript runtime +// today. Keep them in the strict profile so the known divergence remains easy +// to reproduce without making the stable compatibility gate permanently red. +const stableCommandArbitraries = commandArbitraries.filter( + (_, index) => index !== 6 && index !== 7, +); + +export function scenarioArbitrary(maxCommands, { strict = false } = {}) { + const commands = strict ? commandArbitraries : stableCommandArbitraries; + // Repeat edits and transport commands to bias generated traces toward states + // where multiple container kinds and peers interact. + const command = fc.oneof( + ...commands, + ...commands.slice(0, strict ? 18 : 16), + commandArbitraries[18], + commandArbitraries[19], + ); + return fc.record({ + schemaVersion: fc.constant(1), + peerCount: fc.integer({ min: 2, max: 4 }), + commands: fc.array(command, { minLength: 1, maxLength: maxCommands }), + }); +} + +export function assertScenario(value) { + if (value === null || typeof value !== "object") { + throw new TypeError("scenario must be an object"); + } + if (value.schemaVersion !== 1) { + throw new RangeError(`unsupported scenario schema ${String(value.schemaVersion)}`); + } + if (!Number.isSafeInteger(value.peerCount) || value.peerCount < 2) { + throw new RangeError("scenario peerCount must be at least two"); + } + if (!Array.isArray(value.commands)) { + throw new TypeError("scenario commands must be an array"); + } + return value; +} diff --git a/loro-js/package.json b/loro-js/package.json index 12ae6541f..b12295638 100644 --- a/loro-js/package.json +++ b/loro-js/package.json @@ -50,6 +50,10 @@ "build": "vp pack", "check": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm test && pnpm build", "clean": "rm -rf dist", + "fuzz:interop": "pnpm build && node fuzz/run-interop-fuzz.mjs", + "fuzz:interop:ci": "pnpm build && node fuzz/run-interop-fuzz.mjs --ci", + "fuzz:interop:native": "cargo build -p fuzz --bin loro_js_interop_driver && LORO_INTEROP_NATIVE_DRIVER=../target/debug/loro_js_interop_driver pnpm fuzz:interop -- --corpus-only", + "fuzz:interop:strict": "pnpm build && node fuzz/run-interop-fuzz.mjs --strict", "format": "vp fmt", "format:check": "vp fmt --check", "fixtures:rewrite": "pnpm build && node scripts/rewrite-rust-fixture.mjs", @@ -60,6 +64,7 @@ "devDependencies": { "@types/node": "^22.0.0", "@typescript/native-preview": "7.0.0-dev.20260506.1", + "fast-check": "^4.9.0", "typescript": "^5.6.3", "vite": "8.0.10", "vite-plus": "0.1.20", diff --git a/loro-js/src/codec/serde-columnar.ts b/loro-js/src/codec/serde-columnar.ts index b5ba8fd6c..e1fc3f6cb 100644 --- a/loro-js/src/codec/serde-columnar.ts +++ b/loro-js/src/codec/serde-columnar.ts @@ -1,6 +1,13 @@ import { ByteReader, ByteWriter } from "./bytes"; import { LoroDecodeError, LoroEncodeError, decodeAssert, encodeAssert } from "./errors"; -import { I64_MAX, I64_MIN, U64_MAX, readUleb128, readUlebNumber, writeUleb128 } from "./leb128"; +import { + I64_MAX, + I64_MIN, + U64_MAX, + readUleb128, + readUlebNumber, + writeUleb128, +} from "./leb128"; const U32_MAX = 0xffff_ffff; const I32_MIN = -0x8000_0000; @@ -774,7 +781,10 @@ function readPostcardI128NumberOrBigInt(reader: ByteReader): number | bigint { return encoded % 2 === 0 ? encoded / 2 : -(encoded + 1) / 2; } const value = (encodedBigInt >> 1n) ^ -(encodedBigInt & 1n); - decodeAssert(value >= I128_MIN && value <= I128_MAX, "postcard i128 is out of range"); + decodeAssert( + value >= I128_MIN && value <= I128_MAX, + "postcard i128 is out of range", + ); return value; } multiplier *= 128; diff --git a/loro-js/src/codec/state-snapshot.ts b/loro-js/src/codec/state-snapshot.ts index 34471ed8b..50c0d6dac 100644 --- a/loro-js/src/codec/state-snapshot.ts +++ b/loro-js/src/codec/state-snapshot.ts @@ -898,10 +898,7 @@ function assertDecodedPeerIndex(index: bigint, peerCount: bigint, label: string) } function assertEncodedPeerIndex(index: bigint, peerCount: bigint, label: string): void { - encodeAssert( - index >= 0n && index < peerCount, - `${label} peer index out of range`, - ); + encodeAssert(index >= 0n && index < peerCount, `${label} peer index out of range`); } function validateDecodedText( @@ -975,8 +972,7 @@ function validateEncodedTree(state: TreeStateSnapshot): void { assertEncodedPeerIndex(node.peerIndex, peerCount, "tree node ID"); assertEncodedPeerIndex(node.lastSetPeerIndex, peerCount, "tree last-set ID"); encodeAssert( - node.parentIndexPlusTwo >= 0n && - node.parentIndexPlusTwo <= parentIndexBound, + node.parentIndexPlusTwo >= 0n && node.parentIndexPlusTwo <= parentIndexBound, "tree parent index out of range", ); encodeAssert( diff --git a/loro-js/src/runtime/containers.ts b/loro-js/src/runtime/containers.ts index 69e5fa726..aaaec1d52 100644 --- a/loro-js/src/runtime/containers.ts +++ b/loro-js/src/runtime/containers.ts @@ -2124,6 +2124,11 @@ export interface TreeNodeRecord { readonly data: LoroMap; } +const DELETED_TREE_ROOT_ID: CodecId = { + peer: 18_446_744_073_709_551_615n, + counter: 2_147_483_647, +}; + interface TreeJsonValue { readonly id: TreeID; readonly parent: TreeID | null; @@ -2139,11 +2144,9 @@ export class LoroTree< readonly _nodes = new Map(); // Parent-keyed children indexes, keyed numerically (peer -> counter) so tree // reads/writes never stringify ids. Root children live in `_rootChildren`. - readonly _children = new Map< - bigint, - Map> - >(); + readonly _children = new Map>>(); private _rootChildren: OrderedIndex | undefined; + private _deletedRoots: OrderedIndex | undefined; _fractionalIndexEnabled = true; kind(): "Tree" { @@ -2176,7 +2179,11 @@ export class LoroTree< } isNodeDeleted(target: TreeID): boolean { - return this._nodes.get(target)?.deleted ?? false; + const node = this._nodes.get(target); + if (node === undefined) { + throw new RangeError(`tree node ${target} does not exist`); + } + return this._isRecordDeleted(node); } enableFractionalIndex(jitter = 0): void { @@ -2200,13 +2207,17 @@ export class LoroTree< } getNodes(options: { withDeleted?: boolean } = {}): LoroTreeNode[] { - const output: LoroTreeNode[] = []; - for (const record of this._nodes.values()) { - if (options.withDeleted === true || !record.deleted) { - output.push(new LoroTreeNode(this, record.id)); - } + const records = this._nodesUnder(this._childrenOf(undefined)); + if (options.withDeleted === true) { + records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); } - return output; + return records.map((record) => new LoroTreeNode(this, record.id)); + } + + _recordsForSnapshot(): TreeNodeRecord[] { + const records = this._nodesUnder(this._childrenOf(undefined)); + records.push(...this._nodesUnder(this._deletedRoots?.values() ?? [])); + return records; } nodes(): LoroTreeNode[] { @@ -2260,9 +2271,7 @@ export class LoroTree< } _rootJsonValuesRange(start: number, end: number): TreeJsonValue[] { - return ( - this._childrenIndexFor(undefined)?.valuesRange(start, end) ?? [] - ).map( + return (this._childrenIndexFor(undefined)?.valuesRange(start, end) ?? []).map( (record, offset) => this._recordToValue(record, start + offset) as TreeJsonValue, ); } @@ -2310,7 +2319,12 @@ export class LoroTree< _setRecord(record: TreeNodeRecord): void { this._nodes.set(formatTreeId(record.id), record); record.data._setParentBinding(this, { kind: "tree", record }); - if (!record.deleted) this._childrenIndex(record.parent).add(record); + if (record.deleted) { + record.parent = DELETED_TREE_ROOT_ID; + this._deletedRootsIndex().add(record); + } else { + this._childrenIndex(record.parent).add(record); + } } _updateRecord( @@ -2320,7 +2334,11 @@ export class LoroTree< writer: LastWriter, lastMoveId: CodecId, ): void { - if (!record.deleted) this._childrenIndexFor(record.parent)?.delete(record); + if (record.deleted) { + this._deletedRoots?.delete(record); + } else { + this._childrenIndexFor(record.parent)?.delete(record); + } record.parent = parent; record.position = position; record.deleted = false; @@ -2329,22 +2347,49 @@ export class LoroTree< this._childrenIndex(parent).add(record); } - _deleteRecord(record: TreeNodeRecord, writer: LastWriter): void { - if (!record.deleted) this._childrenIndexFor(record.parent)?.delete(record); + _deleteRecord(record: TreeNodeRecord, writer: LastWriter, lastMoveId: CodecId): void { + if (record.deleted) { + this._deletedRoots?.delete(record); + } else { + this._childrenIndexFor(record.parent)?.delete(record); + } + record.parent = DELETED_TREE_ROOT_ID; + record.position = Uint8Array.of(0x80); record.deleted = true; record.writer = writer; + record.lastMoveId = lastMoveId; + this._deletedRootsIndex().add(record); } _removeRecord(record: TreeNodeRecord): void { - if (!record.deleted) this._childrenIndexFor(record.parent)?.delete(record); + if (record.deleted) { + this._deletedRoots?.delete(record); + } else { + this._childrenIndexFor(record.parent)?.delete(record); + } record.deleted = true; this._nodes.delete(formatTreeId(record.id)); } _indexOf(record: TreeNodeRecord): number { + if (record.deleted) return -1; return this._childrenIndexFor(record.parent)?.indexOf(record) ?? -1; } + _isRecordDeleted(record: TreeNodeRecord): boolean { + const visited = new Set(); + let current: TreeNodeRecord | undefined = record; + while (current !== undefined) { + if (current.deleted) return true; + if (current.parent === undefined) return false; + const key = formatTreeId(current.parent); + if (visited.has(key)) return true; + visited.add(key); + current = this._nodes.get(key); + } + return true; + } + _recordToValue( record: TreeNodeRecord, index = this._indexOf(record), @@ -2405,6 +2450,18 @@ export class LoroTree< this._nodes.clear(); this._children.clear(); this._rootChildren = undefined; + this._deletedRoots = undefined; + } + + private _nodesUnder(roots: readonly TreeNodeRecord[]): TreeNodeRecord[] { + const output: TreeNodeRecord[] = []; + const queue = roots.slice(); + for (let index = 0; index < queue.length; index += 1) { + const record = queue[index]!; + output.push(record); + queue.push(...this._childrenOf(record.id)); + } + return output; } private _childrenIndexFor( @@ -2431,6 +2488,10 @@ export class LoroTree< } return index; } + + private _deletedRootsIndex(): OrderedIndex { + return (this._deletedRoots ??= new OrderedIndex(compareTreeRecords)); + } } export class LoroTreeNode = Record> { @@ -2462,11 +2523,15 @@ export class LoroTreeNode = Record): void { const parent = target.parent(); - this.move(parent, target.index() + 1); + const index = target.index(); + if (index === undefined) throw new RangeError(`tree node ${target.id} is deleted`); + this.move(parent, index + 1); } moveBefore(target: LoroTreeNode): void { - this.move(target.parent(), target.index()); + const index = target.index(); + if (index === undefined) throw new RangeError(`tree node ${target.id} is deleted`); + this.move(target.parent(), index); } parent(): LoroTreeNode | undefined { @@ -2484,19 +2549,21 @@ export class LoroTreeNode = Record 0xffff ? 2 : 1; } container.delete(prefixLength, currentLength - prefixLength - suffixLength); - container.insert( - prefixLength, - target.slice(prefixLength, targetLength - suffixLength), - ); + container.insert(prefixLength, target.slice(prefixLength, targetLength - suffixLength)); } diff --git a/loro-js/src/runtime/document.ts b/loro-js/src/runtime/document.ts index 3a7e5ac42..1970be88a 100644 --- a/loro-js/src/runtime/document.ts +++ b/loro-js/src/runtime/document.ts @@ -236,6 +236,7 @@ type PendingEventState = readonly value: unknown; } >; + readonly changed: Set; } | { readonly kind: "counter"; readonly before: number } | { @@ -660,6 +661,9 @@ export class LoroDoc = Record = Record = Record = Record { + const afterRecord = container._entries.get(content.key); + const afterVisible = afterRecord !== undefined && !afterRecord.deleted; + const afterValue = afterVisible + ? cloneRuntimeValue(afterRecord.value) + : undefined; + if ( + beforeVisible !== afterVisible || + !eventValuesEqual(beforeValue, afterValue) + ) { + state.changed.add(content.key); + } + }; } } else if (container instanceof LoroTree) { if ( @@ -2903,7 +2927,7 @@ export class LoroDoc = Record = Record(), + changed: new Set(), }; recording.eventStates.set(container.id, state); return state; @@ -3021,6 +3046,7 @@ export class LoroDoc = Record = Record = Record = Record = Record = Record Number(left.deleted) - Number(right.deleted), - ); + // Rust's fast-tree decoder historically accepted the canonical snapshot + // order: live roots/subtrees first, then deleted roots/subtrees, with each + // sibling group ordered by fractional position and last-move writer. + // Preserve that order so snapshots remain byte-level interoperable and so + // parent-local ordered insertion never sees an insertion-order permutation. + const records = container._recordsForSnapshot(); const peers: bigint[] = []; const peerIndices = new Map(); const peerIndex = (peer: bigint): bigint => { @@ -5478,19 +5515,18 @@ export class LoroDoc = Record = Record (incomingVersion.get(peer) ?? 0)) { + incomingVersion.set(peer, counter); + } + } + } + + // Rust calculates a concurrent import diff by retreating to the common + // ancestor and replaying both branches. Tree operations from the local-only + // branch can therefore produce visible reindex moves even though those + // operations are not present in the imported update. Capture those subjects + // so the incremental TS merge reports the same net transition. + for (const { change } of this.#recordsInVersionRange( + incomingVersion, + beforeVersion, + )) { + for (const operation of change.operations) { + const content = operation.content; + if ( + content.type !== "tree-create" && + content.type !== "tree-move" && + content.type !== "tree-delete" + ) { + continue; + } + const container = this.#containers.get(this.#containerKey(operation.container)); + if (!(container instanceof LoroTree)) continue; + const state = this.#treeEventState(recording, container); + const nodeId = formatTreeId(content.subject); + if (state.originals.has(nodeId)) continue; + const record = container._nodes.get(nodeId); + state.originals.set( + nodeId, + record === undefined || container._isRecordDeleted(record) + ? undefined + : this.#treeEventNode(container, record), + ); + } + } + } + #captureContainerEventValues(ids: ReadonlySet): Map { const values = new Map(); for (const id of ids) { @@ -5897,12 +5987,34 @@ export class LoroDoc = Record = new Map(), ): void { if (changed.size === 0 || !this.#hasEventSubscribers()) return; - const events: LoroEvent[] = []; + const expandedChanged = new Set(changed); + const revived = new Set(); + const calculatedDiffs = new Map(preparedDiffs); for (const id of changed) { const container = this.#containers.get(id); - if (container === undefined) continue; + if (!(container instanceof LoroTree) || this._isContainerDeleted(container)) { + continue; + } const diff = - preparedDiffs.get(id) ?? containerDiff(container, beforeValues.get(id)); + calculatedDiffs.get(id) ?? containerDiff(container, beforeValues.get(id)); + calculatedDiffs.set(id, diff); + if (diff.type !== "tree") continue; + for (const item of diff.diff) { + if (item.action !== "create") continue; + const record = container._nodes.get(item.target); + if (record === undefined || container._isRecordDeleted(record)) continue; + expandedChanged.add(record.data.id); + revived.add(record.data.id); + } + } + + const events: LoroEvent[] = []; + for (const id of expandedChanged) { + const container = this.#containers.get(id); + if (container === undefined || this._isContainerDeleted(container)) continue; + const diff = + calculatedDiffs.get(id) ?? + containerDiff(container, revived.has(id) ? undefined : beforeValues.get(id)); if (isEmptyContainerDiff(diff)) continue; events.push({ target: id as ContainerID, diff, path: containerPath(container) }); } @@ -6194,27 +6306,6 @@ function changeLength(change: DecodedChange): number { return length; } -function advanceUnicodeScalars(value: string, start: number, length: number): number { - let offset = start; - for (let index = 0; index < length; index += 1) { - if (offset >= value.length) { - throw new Error("text snapshot span exceeds the decoded text"); - } - const first = value.charCodeAt(offset); - offset += 1; - if ( - first >= 0xd800 && - first <= 0xdbff && - offset < value.length && - value.charCodeAt(offset) >= 0xdc00 && - value.charCodeAt(offset) <= 0xdfff - ) { - offset += 1; - } - } - return offset; -} - function validateStateSnapshotTextIds( store: StateSnapshotStore, ): ReadonlyMap { @@ -7835,7 +7926,7 @@ function containerEventValue(container: LoroContainer): unknown { return container.getNodes().map((node) => ({ id: node.id, parent: node.parent()?.id, - index: node.index(), + index: node.index()!, fractionalIndex: node.fractionalIndex() ?? "", })) satisfies TreeEventNode[]; } @@ -7961,12 +8052,7 @@ function splitsSurrogatePair(value: string, at: number): boolean { if (at <= 0 || at >= value.length) return false; const before = value.charCodeAt(at - 1); const after = value.charCodeAt(at); - return ( - before >= 0xd800 && - before <= 0xdbff && - after >= 0xdc00 && - after <= 0xdfff - ); + return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff; } function stringDelta(before: string, after: string): Delta[] { @@ -8091,7 +8177,9 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { if (left instanceof Uint8Array && right instanceof Uint8Array) { return bytesEqual(left, right); } + if (left instanceof Uint8Array || right instanceof Uint8Array) return false; if (isContainer(left) && isContainer(right)) return left.id === right.id; + if (isContainer(left) || isContainer(right)) return false; if (Array.isArray(left) && Array.isArray(right)) { if (left.length !== right.length) return false; for (let index = 0; index < left.length; index += 1) { @@ -8099,6 +8187,7 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { } return true; } + if (Array.isArray(left) || Array.isArray(right)) return false; if (left instanceof Map && right instanceof Map) { if (left.size !== right.size) return false; for (const [key, value] of left) { @@ -8106,6 +8195,7 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { } return true; } + if (left instanceof Map || right instanceof Map) return false; if ( typeof left === "object" && left !== null && @@ -8122,7 +8212,7 @@ function eventValuesEqual(left: unknown, right: unknown): boolean { } } let rightCount = 0; - for (const key in rightRecord) rightCount += 1; + for (const _key in rightRecord) rightCount += 1; return leftCount === rightCount; } return false; diff --git a/loro-js/src/runtime/event-diff.ts b/loro-js/src/runtime/event-diff.ts index 6413f47a2..645f5617c 100644 --- a/loro-js/src/runtime/event-diff.ts +++ b/loro-js/src/runtime/event-diff.ts @@ -61,7 +61,8 @@ export class SequenceEventDiff { const inserted = this.#newNode({ kind: "text", value, - attributes: attributes !== undefined && hasAttributes(attributes) ? attributes : undefined, + attributes: + attributes !== undefined && hasAttributes(attributes) ? attributes : undefined, }); this.#root = mergeNodes(mergeNodes(left, inserted), right); } @@ -242,7 +243,11 @@ function splitPiece(piece: Piece, position: number): [Piece, Piece] { } if (piece.kind === "text") { return [ - { kind: "text", value: piece.value.slice(0, position), attributes: piece.attributes }, + { + kind: "text", + value: piece.value.slice(0, position), + attributes: piece.attributes, + }, { kind: "text", value: piece.value.slice(position), attributes: piece.attributes }, ]; } diff --git a/loro-js/src/runtime/jsonpath.ts b/loro-js/src/runtime/jsonpath.ts index 0db42104a..8d715651c 100644 --- a/loro-js/src/runtime/jsonpath.ts +++ b/loro-js/src/runtime/jsonpath.ts @@ -565,10 +565,7 @@ class FilterParser { #parseComparison(): unknown { const left = this.#parseUnary(); const token = this.#peek(); - if ( - token.kind !== "operator" || - !COMPARISON_OPERATORS.includes(token.value) - ) { + if (token.kind !== "operator" || !COMPARISON_OPERATORS.includes(token.value)) { return left; } this.#index += 1; diff --git a/loro-js/tests/event-diff.test.ts b/loro-js/tests/event-diff.test.ts index 1508556cb..224b9d0e3 100644 --- a/loro-js/tests/event-diff.test.ts +++ b/loro-js/tests/event-diff.test.ts @@ -238,6 +238,46 @@ describe("incremental event diffs", () => { ], }); }); + + test("reports local tree reindexing during a concurrent import", () => { + const incoming = new LoroDoc(); + incoming.setPeerId(1); + const incomingNode = incoming.getTree("tree").createNode(); + incoming.commit(); + + const target = new LoroDoc(); + target.setPeerId(2); + const localNode = target.getTree("tree").createNode(); + target.commit(); + let batch: LoroEventBatch | undefined; + target.subscribe((event) => { + batch = event; + }); + + target.import(incoming.export({ mode: "update" })); + + expect(batch?.events[0]?.diff).toEqual({ + type: "tree", + diff: [ + { + target: localNode.id, + action: "move", + parent: undefined, + index: 1, + fractionalIndex: localNode.fractionalIndex(), + oldParent: undefined, + oldIndex: 0, + }, + { + target: incomingNode.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: incomingNode.fractionalIndex(), + }, + ], + }); + }); }); function applyTextDelta(original: string, delta: readonly Delta[]): string { diff --git a/loro-js/tests/runtime.test.ts b/loro-js/tests/runtime.test.ts index 7b7902ba5..4c75e6177 100644 --- a/loro-js/tests/runtime.test.ts +++ b/loro-js/tests/runtime.test.ts @@ -527,6 +527,73 @@ describe("loro-wasm-compatible runtime", () => { }); }); + test("emits the final map value after a delete and restore in one commit", () => { + const doc = new LoroDoc(); + doc.setPeerId(2); + const map = doc.getMap("map"); + map.set("title", {}); + doc.commit(); + + const batches: LoroEventBatch[] = []; + doc.subscribe((event) => batches.push(event)); + map.delete("title"); + map.set("title", {}); + doc.attach(); + + expect(batches).toHaveLength(1); + expect(batches[0]?.events).toEqual([ + { + target: map.id, + diff: { type: "map", updated: { title: {} } }, + path: ["map"], + }, + ]); + }); + + test("preserves visible map transitions but suppresses same-value import winners", () => { + const source = new LoroDoc(); + source.setPeerId(2); + const sourceMap = source.getMap("map"); + sourceMap.set("title", {}); + source.commit(); + + const target = new LoroDoc(); + target.setPeerId(1); + target.import(source.export({ mode: "update" })); + const batches: LoroEventBatch[] = []; + target.subscribe((event) => batches.push(event)); + + sourceMap.delete("title"); + sourceMap.set("title", {}); + source.commit(); + target.import(source.export({ mode: "update" })); + expect(batches.map((batch) => batch.events[0]?.diff)).toEqual([ + { type: "map", updated: { title: {} } }, + ]); + + const concurrent = new LoroDoc(); + concurrent.setPeerId(3); + concurrent.getMap("map").set("title", {}); + concurrent.commit(); + batches.length = 0; + target.import(concurrent.export({ mode: "update" })); + expect(batches).toEqual([]); + }); + + test("does not emit changes for tree metadata deleted in the same commit", () => { + const doc = new LoroDoc(); + const tree = doc.getTree("tree"); + const batches: LoroEventBatch[] = []; + doc.subscribe((event) => batches.push(event)); + + const node = tree.createNode(); + node.data.set("value", []); + tree.delete(node.id); + doc.commit(); + + expect(batches).toEqual([]); + }); + test("calculates raw and JSON-compatible diffs without moving document state", () => { const doc = new LoroDoc(); const map = doc.getMap("map"); @@ -1610,6 +1677,27 @@ describe("loro-wasm-compatible runtime", () => { expect(doc.opCount()).toBe(opCount); }); + test("does not confuse empty value shapes when detecting no-op edits", () => { + const doc = new LoroDoc(); + const map = doc.getMap("map"); + const movable = doc.getMovableList("movable"); + map.set("value", []); + movable.push([]); + doc.commit(); + const opCount = doc.opCount(); + + map.set("value", {}); + movable.set(0, {}); + doc.commit(); + map.set("value", new Uint8Array()); + movable.set(0, new Uint8Array()); + doc.commit(); + + expect(map.get("value")).toEqual(new Uint8Array()); + expect(movable.get(0)).toEqual(new Uint8Array()); + expect(doc.opCount()).toBe(opCount + 4); + }); + test("resurfaces preserved state and switches mergeable kinds", () => { const doc = new LoroDoc(); const root = doc.getMap("state"); @@ -1749,15 +1837,60 @@ describe("loro-wasm-compatible runtime", () => { expect(tree.isFractionalIndexEnabled()).toBe(true); expect(() => tree.enableFractionalIndex(256)).toThrow(/unsigned byte/u); - const deletedFractionalIndex = child.fractionalIndex(); + const lastMoveBeforeDelete = child.getLastMoveId(); tree.delete(child.id); expect(tree.isNodeDeleted(child.id)).toBe(true); + expect(() => tree.isNodeDeleted("999@999")).toThrow(/does not exist/u); expect(tree.getNodes({ withDeleted: true })).toHaveLength(2); expect(root.toJSON()).toMatchObject({ parent: undefined, fractionalIndex: "80" }); + expect(child.parent()?.id).toBe("2147483647@18446744073709551615"); + expect(child.index()).toBeUndefined(); + expect(child.fractionalIndex()).toBeUndefined(); + expect(child.getLastMoveId()).not.toEqual(lastMoveBeforeDelete); expect(child.toJSON()).toMatchObject({ parent: "2147483647@18446744073709551615", index: 0, - fractionalIndex: deletedFractionalIndex, + fractionalIndex: "80", + }); + }); + + test("keeps deleted tree subtrees out of the live forest and snapshots them", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + tree.enableFractionalIndex(0); + const liveRoot = tree.createNode(); + const deletedRoot = tree.createNode(); + const deletedChild = deletedRoot.createNode(); + + expect(deletedRoot.toJSON().fractionalIndex).toBe("8180"); + tree.delete(deletedRoot.id); + expect(tree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); + expect(tree.getNodes({ withDeleted: true }).map((node) => node.id)).toEqual([ + liveRoot.id, + deletedRoot.id, + deletedChild.id, + ]); + expect(tree.isNodeDeleted(deletedChild.id)).toBe(true); + expect(deletedChild.isDeleted()).toBe(true); + expect(deletedChild.data.isDeleted()).toBe(true); + expect(deletedRoot.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", + }); + + doc.commit(); + const restored = new LoroDoc(); + restored.import(doc.export({ mode: "snapshot" })); + const restoredTree = restored.getTree("tree"); + expect(restoredTree.getNodes().map((node) => node.id)).toEqual([liveRoot.id]); + expect(restoredTree.isNodeDeleted(deletedRoot.id)).toBe(true); + expect(restoredTree.isNodeDeleted(deletedChild.id)).toBe(true); + expect(restoredTree.getNodeByID(deletedRoot.id)?.toJSON()).toMatchObject({ + parent: "2147483647@18446744073709551615", + index: 0, + fractionalIndex: "80", }); }); diff --git a/loro-js/tests/version-transition.test.ts b/loro-js/tests/version-transition.test.ts index 2a254f518..6a8f61667 100644 --- a/loro-js/tests/version-transition.test.ts +++ b/loro-js/tests/version-transition.test.ts @@ -4,6 +4,75 @@ import { LoroDoc } from "../src/runtime/document"; import type { LoroEventBatch } from "../src/runtime/types"; describe("indexed version transitions", () => { + test("reports tree checkout deletions at their original sibling indices", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const emptyFrontiers = doc.frontiers(); + const tree = doc.getTree("tree"); + const first = tree.createNode(); + const second = tree.createNode(); + doc.commit(); + let batch: LoroEventBatch | undefined; + doc.subscribe((event) => { + batch = event; + }); + + doc.checkout(emptyFrontiers); + + expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ + type: "tree", + diff: [ + { + target: first.id, + action: "delete", + oldParent: undefined, + oldIndex: 0, + }, + { + target: second.id, + action: "delete", + oldParent: undefined, + oldIndex: 1, + }, + ], + }); + }); + + test("revives tree metadata events when checkout restores a node", () => { + const doc = new LoroDoc(); + doc.setPeerId(1); + const tree = doc.getTree("tree"); + const node = tree.createNode(); + node.data.set("value", {}); + doc.commit(); + const liveFrontiers = doc.frontiers(); + tree.delete(node.id); + doc.commit(); + let batch: LoroEventBatch | undefined; + doc.subscribe((event) => { + batch = event; + }); + + doc.checkout(liveFrontiers); + + expect(batch?.events.find(({ target }) => target === node.data.id)?.diff).toEqual({ + type: "map", + updated: { value: {} }, + }); + expect(batch?.events.find(({ target }) => target === tree.id)?.diff).toEqual({ + type: "tree", + diff: [ + { + target: node.id, + action: "create", + parent: undefined, + index: 0, + fractionalIndex: node.fractionalIndex(), + }, + ], + }); + }); + test("retreats and restores indexed container state", () => { const doc = new LoroDoc(); doc.setPeerId(1); diff --git a/package.json b/package.json index 7c8d85ff6..8ebba034f 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test-esbuild": "pnpm --dir examples/test-esbuild install --ignore-workspace --prefer-offline && pnpm --filter loro-test-esbuild serve", "test-bundlers": "pnpm --dir examples/bundler-smoke-tests run test:fast", "test-bundlers-next": "pnpm --dir examples/bundler-smoke-tests run test:next", + "test-loro-js-interop": "pnpm --filter loro-js fuzz:interop:ci && pnpm --filter loro-js fuzz:interop:native", "run-fuzz-corpus": "node ./scripts/cargo-fuzz-run.mjs all -- -max_total_time=1", "fix": "cargo clippy --fix --features=test_utils", "vet": "cargo vet", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 549f7b79a..79d3d2dfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -180,6 +180,9 @@ importers: '@typescript/native-preview': specifier: 7.0.0-dev.20260506.1 version: 7.0.0-dev.20260506.1 + fast-check: + specifier: ^4.9.0 + version: 4.9.0 typescript: specifier: ^5.6.3 version: 5.6.3 @@ -2466,6 +2469,10 @@ packages: resolution: {integrity: sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==} engines: {node: '>=4'} + fast-check@4.9.0: + resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -3326,6 +3333,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + quansync@0.2.11: resolution: {integrity: sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==} @@ -6075,6 +6085,10 @@ snapshots: iconv-lite: 0.4.24 tmp: 0.0.33 + fast-check@4.9.0: + dependencies: + pure-rand: 8.4.2 + fast-deep-equal@3.1.3: {} fast-diff@1.1.2: {} @@ -6915,6 +6929,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + quansync@0.2.11: {} queue-microtask@1.2.3: {}