diff --git a/.changeset/fallible-unparsable-history.md b/.changeset/fallible-unparsable-history.md new file mode 100644 index 000000000..19d87d17e --- /dev/null +++ b/.changeset/fallible-unparsable-history.md @@ -0,0 +1,11 @@ +--- +"loro-crdt": minor +--- + +Return decode errors when fallible history operations first read an unreadable change block, preserving document state instead of panicking during lazy DAG loading. + +Keep attached imports atomic even after all DAG headers have been loaded: unparsed change-block bodies still require rollback protection. Share the existing version snapshot between rollback journals to avoid two extra copies of the version vector. Legacy infallible range readers continue skipping only the damaged block while recording the failure. + +`state_vv` / `version`, JSON update export, cursor lookup, `apply_diff` rollback, shallow imports, and `fork_at` return `DecodeError` instead of panicking or wrapping the failure as `Unknown`. `state_vv` and `export_json_updates` now return `LoroResult`. `try_state_vv`, `try_export_json_updates`, and `try_get_cursor_pos` report the same failure. `CannotFindRelativePosition` is `#[non_exhaustive]` and gains `HistoryUnreadable`. `LoroEncodeError` gains `DecodeError`. + +Rust compatibility: `ChangeTravelError` gains `HistoryUnreadable(LoroError)` and is now `#[non_exhaustive]`; downstream exhaustive matches must add a wildcard arm. `FrontiersNotIncluded` changes from a unit struct to a struct with a private field plus a same-named constant. Construction via that constant still compiles, but an exhaustive `match` of `Err(FrontiersNotIncluded)` does not (`E0004`): the `history_error: Some(_)` pattern cannot be written from another crate (`E0451`) because the field is private. Downstream must use `Err(_)`. An unreadable-history error is not equal to the `FrontiersNotIncluded` constant. diff --git a/Cargo.lock b/Cargo.lock index c2b079b9e..26f9a190f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1606,6 +1606,7 @@ dependencies = [ "serde_json", "serial_test", "tracing", + "xxhash-rust", ] [[package]] @@ -1935,7 +1936,7 @@ checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" [[package]] name = "loro-wasm" -version = "1.16.3" +version = "1.16.4" dependencies = [ "js-sys", "loro-common 1.16.0", diff --git a/context/arena-parent-links.md b/context/arena-parent-links.md index 92750c3ca..d9613b3b4 100644 --- a/context/arena-parent-links.md +++ b/context/arena-parent-links.md @@ -1,6 +1,6 @@ # Arena Parent Links -Verified against code 2026-10-01. +Verified against code 2026-10-09. `SharedArena` (`crates/loro-internal/src/arena.rs`) stores each container's parent. Liveness (`DocState::is_deleted`), paths (`DocState::get_path`, @@ -105,62 +105,145 @@ No path leads to it, so callers treat it like a deleted container: ## A block that cannot be parsed -Snapshot import validates the KV checksums (`ChangeStore::import_all`), so a -lazily loaded block that fails to parse is forged or truncated external input, -not an internal invariant. The resolver runs under the state lock from queries -that return `bool` or `Option` (`is_deleted`, `has_container`, `get_path`), so -it has no `Err` to return, and a panic there unwinds under the locks and traps -the WASM instance. Until 2026-09-30 it panicked. +Snapshot import validates the KV checksums (`ChangeStore::import_all`), but +loads full-snapshot history lazily. A forged or truncated block with recomputed +checksums can therefore be accepted before its contents are read. Honest data +does not produce such a block; a decode/parse failure is invalid external input, +not an impossible internal invariant. Every ordinary history reader that fails to decode or parse a block records it in `ChangeStore::parse_failures` (a leaf lock; the first failure is kept) and answers "no such change", as the readers other than the resolver always did. -The resolver answers `CreatorOp::Corrupt`, which the arena treats like `Absent` -for that one lookup: the container reads as deleted, the ID as not a container. - -That answer may be wrong, so `OpLog::check_history_parsable` turns the record -into `DecodeError("cannot parse change block ...")`. **Only public entry points -that return a `Result` call it:** - -- `LoroDoc::checkout`, `diff` and `revert_to` (through `diff_events`), and - `import` (every path, through - `import_changes_and_apply_delta_to_state_if_needed`), before they start. -- `LoroDoc::export` (and so `fork_at` and `merge`), before it starts and again - when it is done, so an export that is the first to read the block fails - instead of returning bytes without the block's changes. - -Do not move the check into `_checkout_without_emitting` or the exporters. -`undo` (`calc_diff`), `checkout_to_latest`, and `fork` on a detached doc call -them and `unwrap` the result, so a check there turns the record into a panic -on those paths (the first version of this fix did). `fork` uses -`export_inner(.., false)` / `fork_at_inner(.., false)` for the same reason. - -Limits: - -- The paths without the check behave as before the record existed. They can - still panic when they need the broken block itself: `AppDag` loads its nodes - from the change store and panics with "unparsed vv don't match with change - store" (`loro_dag.rs`, `ensure_lazy_load_node`). -- A read that returns before any failure was recorded is not undone: an - `import` or `checkout` using a general history reader can finish on partial - history when it is the first to read the block. Only `export` checks again at - the end. The direct cold Text comparison in `known_history.rs` falls back to - the ordinary reader on a decoding/eligibility error. Its stricter op-length - and change-boundary checks never record a `parse_failures` entry themselves; - only a failure of the ordinary parser declares local history unparsable. -- Local edits still succeed after a failure was recorded, but they cannot be - exported from this document any more. What can be salvaged is the current - state (`get_deep_value`). - -Tests, in `change_store.rs`: -`the_creator_resolver_reports_an_unparsable_block_instead_of_panicking`, -`every_reader_records_an_unparsable_block`, -`a_doc_with_an_unparsable_block_neither_panics_nor_exports_partial_history` -(a truncated block; the container is one that a healthy history finds), -`an_export_that_finds_an_unparsable_block_fails_instead_of_skipping_it`, and -`a_recorded_parse_failure_does_not_panic_where_no_error_can_be_returned` (the -record is set by hand on a healthy doc, to separate the check from the DAG -panic above). +Legacy `Option` readers answer "no such change". The creator resolver answers +`CreatorOp::Corrupt`, which the arena treats like `Absent` for that one lookup: +`is_deleted`, `has_container`, and `get_path` cannot return an error, so the +container reads as deleted or absent and the ID as not a container. This avoids +panicking under their state lock, but does not establish that the container is +actually absent. + +`OpLog::check_history_parsable` converts the record to +`DecodeError("cannot parse change block ...")`. Public fallible operations check +already recorded failures, and their history reads also propagate failures +**on the first read**, without requiring an earlier query to record them: + +- `AppDag::ensure_lazy_load_node` returns `LoroResult<()>`. If the change store + cannot provide a node, it checks the recorded parse failure and returns it. + If there is no parse failure, an id promised by `unparsed_vv` but absent from + the store is still an impossible internal inconsistency: the original + "unparsed vv don't match with change store" assertion remains. +- DAG version conversion, replay-base selection, causal iteration, and diff + calculation use fallible reads. `checkout`, `diff`, `revert_to`, `fork_at`, + binary/JSON/batch imports, and shallow/state-only/snapshot-at exports propagate + the error before applying state. Attached imports enable their existing + rollback journal when the change store may still hold unparsed **bodies**, + including register-only imports that otherwise need no journal. DAG loading + reads headers only: draining `AppDag::unparsed_vv` does not validate bodies. + `ChangeStoreInner::may_have_unparsed_bodies` is a conservative flag set on + snapshot import/fork, cleared by a successful `visit_all_changes`, and set + again when arena rollback discards parsed bodies. Individual body reads do + not clear it; imports can therefore keep using a journal after all bodies + have been read individually, until a full walk confirms that fact. + The attached import, DAG journal, and change-store journal share one + `Arc` snapshot instead of copying that vector three times. + The DAG's separate `unparsed_vv` checkpoint is still copied; after header + warming it is empty. Initial shallow-snapshot import also + returns the error if reading its root fails, then resets state and op log. +- `ChangeStore::try_iter_changes` validates the requested range before handing + changes to a consumer. Export stops at an unreadable block rather than + feeding a gap into a scratch history or returning partial bytes; the public + export's final history check returns the recorded error. Legacy `iter_blocks` + and `iter_changes` instead record and skip only the bad block, preserving + the healthy portions of the requested range at their infallible boundaries. +- Query signatures that previously returned `Option` or plain values remain + available. Use `try_frontiers_to_vv`, `try_vv_to_frontiers`, + `try_minimize_frontiers`, `try_find_id_spans_between`, `try_get_change` + (`OpLog::try_get_change_at`), `try_get_changed_containers_in`, + `try_state_vv`, `try_export_json_updates`, and `try_get_cursor_pos` to + receive decode errors. `state_vv` clones `OpLog::vv` when the state and op + log frontiers are equal; that vector is cached and does not parse block + bodies. Otherwise it uses `try_frontiers_to_vv`, and returns + `FrontiersNotFound` when a frontier id is absent rather than an empty + version vector. `cmp_frontiers` keeps its `FrontiersNotIncluded` error type, + which can now carry the decode-error message; `travel_change_ancestors` + returns `ChangeTravelError::HistoryUnreadable(LoroError::DecodeError(..))`. + `ChangeTravelError` is now non-exhaustive, so downstream matches need a + wildcard arm. `FrontiersNotIncluded` is a struct with a private field plus a + same-named constant. Construction via the constant still compiles, but an + exhaustive `match` of `Err(FrontiersNotIncluded)` does not (`E0004`): the + `history_error: Some(_)` pattern cannot be written from another crate + (`E0451`) because the field is private. Downstream must use `Err(_)`. An + unreadable-history error is not equal to the constant. + `CannotFindRelativePosition` is non-exhaustive and gains `HistoryUnreadable`. + `LoroEncodeError` gains `DecodeError`, so `fork_at` keeps a decode failure + instead of wrapping it as `Unknown`. WASM queries that already return + `JsResult` use these fallible readers too, including `version`, + `exportJsonUpdates`, `getCursorPos`, `findIdSpansBetween`, `frontiersToVV`, + `vvToFrontiers`, `getChangeAt`, `getChangeAtLamport`, `getOpsInChange`, and + `getChangedContainersIn`; their JS names and return types stay the same. + +A failed fallible operation leaves materialized values, document versions, and +attached/detached status unchanged; loading valid blocks may populate history +caches. The successful DAG lazy-load path does no additional block reads or +parse-failure checks: the error-record lookup is inside the missing-node branch. +The release benchmark `crates/loro/tests/perf_history_lazy_load.rs` measures +fresh snapshot import, cold historical checkout, and concurrent update import. +Its `perf_map_import_many_peers` case measures a one-op causal map import with +1k/10k peers, both cold and after warming every DAG header without parsing old +bodies. Setup is outside the timer. Against pre-review commit `be17aed3`, +three alternating before/after pairs, each with three repeats of 30 imports, +gave median after/before ratios of 1.001/1.011 (cold) and 0.997/0.996 (headers +warmed), on macOS arm64 in release. Sharing the version snapshot removes two +extra full-vector clones from journal creation; opening the journal no longer +adds those O(peers) copies. These timings show no material increase in this +workload, rather than a bound on every import workload. + +Limits and compatibility: + +- Reads that need no history do not eagerly validate every block. For example, + full snapshot import and byte-faithful full snapshot export can copy an + unreadable block before any failure has been recorded. A query that can use + the latest version directly may also succeed without reading it, including + `state_vv` when the state frontiers are the op-log frontiers. +- The legacy helpers used by `undo`, `checkout_to_latest`, and `fork` retain + their infallible boundaries. Their fallible reads record the failure first, + then their existing unwrap boundary may panic if they need the broken history. + A detached `fork` fails outside the source document's locks, leaving fallible + reads on the source able to report the record. Other infallible paths can + still unwind under locks or trap WASM. Do not add blanket record checks to + their shared internal helpers: a recorded failure alone must not break an + operation that can finish using healthy history. A missing node with no + recorded parse failure is still the internal "unparsed vv don't match with + change store" assertion in `ensure_lazy_load_node`. +- The direct cold Text comparison in `known_history.rs` falls back to the + ordinary reader on a decoding/eligibility error. Its stricter op-length and + change-boundary checks never record a `parse_failures` entry themselves; only + a failure of the ordinary parser declares local history unparsable. A read + that returns before any failure was recorded is not undone. +- Local edits and `get_deep_value` can still use the current state after a + failure is recorded. Fallible history operations and public exports then + reject the unreadable history, so the document can no longer be exported. + +Regression tests in `change_store.rs` cover every store reader, the creator +resolver, first DAG reads on tree history, and infallible compatibility. The +internal missing-node assertion is tested in `loro_dag.rs`. +`crates/loro/tests/unregistered_container_parent.rs` +rewrites snapshot bytes through the public KV API and repairs both checksum +layers. Each fallible entry point gets its own fresh document, with the first, +middle, and last peer-1 block truncated; it must return "cannot parse change +block" without a panic, preserve the value/version/status, and leave locks +usable. It also tests initial shallow-import rollback. The regression +`register_only_import_rolls_back_when_only_old_block_headers_were_read` flips +the third byte from the end of the first/middle/last peer-1 block and repairs +the checksums. Both peers' DAG version queries succeed before a concurrent +map import first parses the bad body. The import must return the decode error, +with op log and state frontiers still equal to their original frontiers, +unchanged values/version vector, and an attached document. The healthy-doc test +`a_recorded_parse_failure_does_not_panic_where_no_error_can_be_returned` keeps +manual error records separate from actual unreadable blocks. +`crates/loro-wasm/tests/unparsable_history.test.ts` uses the same corrupted +snapshot as a fixture and checks JS exceptions, unchanged values/versions, +and a usable WASM instance after each first read. Its body-corruption fixture +also verifies a map import's rollback after both peers' header queries succeed. ## Cost of looking up an ID diff --git a/crates/examples/examples/failed_import_rollback_perf.rs b/crates/examples/examples/failed_import_rollback_perf.rs index fd5ab1488..a4031cf5d 100644 --- a/crates/examples/examples/failed_import_rollback_perf.rs +++ b/crates/examples/examples/failed_import_rollback_perf.rs @@ -73,8 +73,12 @@ fn main() { // A failed import: the large update plus a list insert forged out of bounds. other.get_list("list").insert(0, -1).unwrap(); other.commit(); - let mut json = - serde_json::to_value(other.export_json_updates(&base_vv, &other.oplog_vv())).unwrap(); + let mut json = serde_json::to_value( + other + .export_json_updates(&base_vv, &other.oplog_vv()) + .unwrap(), + ) + .unwrap(); let last = json["changes"].as_array_mut().unwrap().last_mut().unwrap()["ops"] .as_array_mut() .unwrap() diff --git a/crates/examples/examples/import_known_history_perf.rs b/crates/examples/examples/import_known_history_perf.rs index 6677ab2f4..603c4e440 100644 --- a/crates/examples/examples/import_known_history_perf.rs +++ b/crates/examples/examples/import_known_history_perf.rs @@ -86,7 +86,9 @@ fn main() { let mut vv = VersionVector::default(); { let replay = LoroDoc::new(); - let json = inc_src.export_json_updates(&Default::default(), &inc_src.oplog_vv()); + let json = inc_src + .export_json_updates(&Default::default(), &inc_src.oplog_vv()) + .unwrap(); for change in json.changes { replay .import_json_updates(json_schema_with(&json.peers, change)) diff --git a/crates/examples/examples/issue_stuck.rs b/crates/examples/examples/issue_stuck.rs index 87463d9b6..3ebe738cb 100644 --- a/crates/examples/examples/issue_stuck.rs +++ b/crates/examples/examples/issue_stuck.rs @@ -27,6 +27,8 @@ fn main() { println!("Imported update"); doc.checkout_to_latest(); doc.check_state_correctness_slow(); - let res = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let res = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); println!("{:#?}", serde_json::to_value(res).unwrap()); } diff --git a/crates/fuzz/src/actor.rs b/crates/fuzz/src/actor.rs index 0bd176889..3acbd04c4 100644 --- a/crates/fuzz/src/actor.rs +++ b/crates/fuzz/src/actor.rs @@ -265,6 +265,7 @@ impl Actor { self.loro.oplog_vv(), self.loro .export_json_updates(&Default::default(), &self.loro.oplog_vv()) + .unwrap() ) }), ); @@ -421,7 +422,8 @@ impl Actor { pub(crate) fn log_json_history(&self) { let json = self .loro - .export_json_updates(&Default::default(), &self.loro.oplog_vv()); + .export_json_updates(&Default::default(), &self.loro.oplog_vv()) + .unwrap(); let string = serde_json::to_string_pretty(&json).unwrap(); info!("vv={:?} json = {}", self.loro.oplog_vv(), string); } diff --git a/crates/fuzz/src/crdt_fuzzer.rs b/crates/fuzz/src/crdt_fuzzer.rs index 9ae6d14fc..bfdd66469 100644 --- a/crates/fuzz/src/crdt_fuzzer.rs +++ b/crates/fuzz/src/crdt_fuzzer.rs @@ -533,13 +533,15 @@ impl CRDTFuzzer { } _ => { info_span!("JsonFormat", from = i, to = j).in_scope(|| { - let a_json = - a_doc.export_json_updates(&b_doc.oplog_vv(), &a_doc.oplog_vv()); + let a_json = a_doc + .export_json_updates(&b_doc.oplog_vv(), &a_doc.oplog_vv()) + .unwrap(); b_doc.import_json_updates(a_json).unwrap(); }); info_span!("JsonFormat", from = j, to = i).in_scope(|| { - let b_json = - b_doc.export_json_updates(&a_doc.oplog_vv(), &b_doc.oplog_vv()); + let b_json = b_doc + .export_json_updates(&a_doc.oplog_vv(), &b_doc.oplog_vv()) + .unwrap(); a_doc.import_json_updates(b_json).unwrap(); }); } @@ -815,8 +817,9 @@ pub fn test_multi_sites_with_gc( } _ => { let synced = info_span!("JsonFormat", from = i, to = j).in_scope(|| { - let a_json = - a_doc.export_json_updates(&b_doc.oplog_vv(), &a_doc.oplog_vv()); + let a_json = a_doc + .export_json_updates(&b_doc.oplog_vv(), &a_doc.oplog_vv()) + .unwrap(); handle_gc_sync_import_result(b_doc.import_json_updates(a_json)) }); can_check_eq &= synced; @@ -824,7 +827,8 @@ pub fn test_multi_sites_with_gc( let synced = info_span!("JsonFormat", from = j, to = i).in_scope(|| { let b_json = b_doc - .export_json_updates(&a_doc.oplog_vv(), &b_doc.oplog_vv()); + .export_json_updates(&a_doc.oplog_vv(), &b_doc.oplog_vv()) + .unwrap(); handle_gc_sync_import_result(a_doc.import_json_updates(b_json)) }); can_check_eq &= synced; diff --git a/crates/fuzz/tests/compatibility.rs b/crates/fuzz/tests/compatibility.rs index 0ccb483aa..3ede506ec 100644 --- a/crates/fuzz/tests/compatibility.rs +++ b/crates/fuzz/tests/compatibility.rs @@ -224,9 +224,11 @@ mod compatibility_with_10_alpha_4 { let updates = serde_json::to_value(doc1.export_json_updates(&Default::default(), &doc1.oplog_vv())) .unwrap(); - let updates_b = - serde_json::to_value(doc2.export_json_updates(&Default::default(), &doc2.oplog_vv())) - .unwrap(); + let updates_b = serde_json::to_value( + doc2.export_json_updates(&Default::default(), &doc2.oplog_vv()) + .unwrap(), + ) + .unwrap(); assert_eq!(updates, updates_b); } diff --git a/crates/fuzz/tests/json.rs b/crates/fuzz/tests/json.rs index e638b0cdc..bc1e99066 100644 --- a/crates/fuzz/tests/json.rs +++ b/crates/fuzz/tests/json.rs @@ -20,7 +20,9 @@ fn unknown_json() { counter.increment(1.).unwrap(); doc.commit(); // json format with counter - let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); // Test1: old version import newer version json if doc_with_unknown .import_json_updates(serde_json::to_string(&json).unwrap()) diff --git a/crates/fuzz/tests/test.rs b/crates/fuzz/tests/test.rs index 48964f35a..ac6585c74 100644 --- a/crates/fuzz/tests/test.rs +++ b/crates/fuzz/tests/test.rs @@ -9636,7 +9636,9 @@ fn failed_shallow_snapshot_import_keeps_text_subscription_recording() { Err(LoroError::SwitchToVersionBeforeShallowRoot) )); - let json = c.export_json_updates(&Default::default(), &c.oplog_vv()); + let json = c + .export_json_updates(&Default::default(), &c.oplog_vv()) + .unwrap(); target.import_json_updates(json).unwrap(); assert_eq!( target.get_text("text").to_string(), diff --git a/crates/loro-common/src/error.rs b/crates/loro-common/src/error.rs index 884a65d97..e6a62ad65 100644 --- a/crates/loro-common/src/error.rs +++ b/crates/loro-common/src/error.rs @@ -144,6 +144,8 @@ pub enum LoroEncodeError { ShallowSnapshotIncompatibleWithOldFormat, #[error("Cannot export shallow snapshot with unknown container type. Please upgrade the Loro version.")] UnknownContainer, + #[error("Decode error: ({0})")] + DecodeError(Box), #[error("Export failed: {0}")] InternalError(Box), } @@ -211,6 +213,7 @@ impl From for LoroError { | LoroEncodeError::UnknownContainer => { LoroError::Unknown(value.to_string().into_boxed_str()) } + LoroEncodeError::DecodeError(msg) => LoroError::DecodeError(msg), LoroEncodeError::InternalError(msg) => LoroError::Unknown(msg), } } @@ -223,6 +226,7 @@ impl From for LoroEncodeError { LoroEncodeError::FrontiersNotFound(format!("{id:?}")) } LoroError::NotFoundError(msg) => LoroEncodeError::FrontiersNotFound(msg.into()), + LoroError::DecodeError(msg) => LoroEncodeError::DecodeError(msg), LoroError::Unknown(msg) => LoroEncodeError::InternalError(msg), other => LoroEncodeError::InternalError(other.to_string().into_boxed_str()), } diff --git a/crates/loro-internal/benches/encode.rs b/crates/loro-internal/benches/encode.rs index 36e5da1fc..96e2ea070 100644 --- a/crates/loro-internal/benches/encode.rs +++ b/crates/loro-internal/benches/encode.rs @@ -118,12 +118,16 @@ mod run { b.bench_function("B4_encode_json_update", |b| { ensure_ran(); b.iter(|| { - let _ = loro.export_json_updates(&Default::default(), &loro.oplog_vv(), true); + let _ = loro + .export_json_updates(&Default::default(), &loro.oplog_vv(), true) + .unwrap(); }) }); b.bench_function("B4_decode_json_update", |b| { ensure_ran(); - let json = loro.export_json_updates(&Default::default(), &loro.oplog_vv(), true); + let json = loro + .export_json_updates(&Default::default(), &loro.oplog_vv(), true) + .unwrap(); b.iter(|| { let store2 = LoroDoc::default(); store2.import_json_updates(json.clone()).unwrap(); diff --git a/crates/loro-internal/examples/encoding.rs b/crates/loro-internal/examples/encoding.rs index 2ac5f309d..96670dcd5 100644 --- a/crates/loro-internal/examples/encoding.rs +++ b/crates/loro-internal/examples/encoding.rs @@ -81,11 +81,11 @@ fn main() { output.len(), ); - let json_updates = serde_json::to_string(&loro.export_json_updates( - &Default::default(), - &loro.oplog_vv(), - true, - )) + let json_updates = serde_json::to_string( + &loro + .export_json_updates(&Default::default(), &loro.oplog_vv(), true) + .unwrap(), + ) .unwrap(); let output = miniz_oxide::deflate::compress_to_vec(json_updates.as_bytes(), 6); println!( diff --git a/crates/loro-internal/examples/encoding_refactored.rs b/crates/loro-internal/examples/encoding_refactored.rs index 709c426c1..44d43b206 100644 --- a/crates/loro-internal/examples/encoding_refactored.rs +++ b/crates/loro-internal/examples/encoding_refactored.rs @@ -25,11 +25,11 @@ fn log_size() { txn.commit().unwrap(); let snapshot = loro.export(ExportMode::Snapshot).unwrap(); let updates = loro.export(ExportMode::all_updates()).unwrap(); - let json_updates = serde_json::to_string(&loro.export_json_updates( - &Default::default(), - &loro.oplog_vv(), - true, - )) + let json_updates = serde_json::to_string( + &loro + .export_json_updates(&Default::default(), &loro.oplog_vv(), true) + .unwrap(), + ) .unwrap(); println!("\n"); println!("Snapshot size={}", snapshot.len()); diff --git a/crates/loro-internal/src/cursor.rs b/crates/loro-internal/src/cursor.rs index f3c355853..6e3a98e0b 100644 --- a/crates/loro-internal/src/cursor.rs +++ b/crates/loro-internal/src/cursor.rs @@ -52,7 +52,13 @@ pub struct AbsolutePosition { pub side: Side, } +/// Why a cursor could not be resolved to an absolute position. +/// +/// Non-exhaustive: `HistoryUnreadable` means the lookup had to read history +/// that cannot be parsed. It is not the same as a missing id. Downstream +/// matches need a wildcard arm. #[derive(Debug, Clone, Copy, thiserror::Error)] +#[non_exhaustive] pub enum CannotFindRelativePosition { #[error("Cannot find relative position. The container is deleted.")] ContainerDeleted, @@ -60,6 +66,9 @@ pub enum CannotFindRelativePosition { HistoryCleared, #[error("Cannot find relative position. The id is not found.")] IdNotFound, + /// The lookup needed a change block that cannot be parsed. + #[error("Cannot find relative position because the document history cannot be read.")] + HistoryUnreadable, } impl Cursor { diff --git a/crates/loro-internal/src/dag.rs b/crates/loro-internal/src/dag.rs index cf4538560..ff194495c 100644 --- a/crates/loro-internal/src/dag.rs +++ b/crates/loro-internal/src/dag.rs @@ -12,10 +12,11 @@ use std::{ fmt::Debug, }; -use loro_common::IdSpanVector; +use loro_common::{IdSpanVector, LoroError, LoroResult}; use rle::{HasLength, Sliceable}; use rustc_hash::{FxHashMap, FxHashSet}; mod iter; +pub(crate) use iter::DagCausalIter; mod mermaid; #[cfg(feature = "test_utils")] mod test; @@ -31,7 +32,7 @@ use crate::{ }; use self::{ - iter::{iter_dag, iter_dag_with_vv, DagCausalIter, DagIterator, DagIteratorVV}, + iter::{iter_dag, iter_dag_with_vv, DagIterator, DagIteratorVV}, mermaid::dag_to_mermaid, }; @@ -53,6 +54,9 @@ pub(crate) trait Dag: Debug { type Node: DagNode; fn get(&self, id: ID) -> Option; + fn try_get(&self, id: ID) -> LoroResult> { + Ok(self.get(id)) + } #[allow(unused)] fn frontier(&self) -> &Frontiers; fn vv(&self) -> &VersionVector; @@ -95,13 +99,34 @@ pub(crate) trait DagUtils: Dag { /// one it can find itself). Finding the single-head cut walks the whole /// DAG, so callers that usually have a better answer should not pay for /// it up front. - fn find_meet_and_mode(&self, a_id: &Frontiers, b_id: &Frontiers) -> (MeetAsBase, DiffMode); - fn latest_single_head_critical_version(&self, a_id: &Frontiers, b_id: &Frontiers) -> Frontiers; + #[cfg(test)] + fn find_meet_and_mode(&self, a_id: &Frontiers, b_id: &Frontiers) -> (MeetAsBase, DiffMode) { + self.try_find_meet_and_mode(a_id, b_id).unwrap() + } + fn try_find_meet_and_mode( + &self, + a_id: &Frontiers, + b_id: &Frontiers, + ) -> LoroResult<(MeetAsBase, DiffMode)>; + #[cfg(test)] + fn latest_single_head_critical_version(&self, a_id: &Frontiers, b_id: &Frontiers) -> Frontiers { + self.try_latest_single_head_critical_version(a_id, b_id) + .unwrap() + } + fn try_latest_single_head_critical_version( + &self, + a_id: &Frontiers, + b_id: &Frontiers, + ) -> LoroResult; /// Slow, should probably only use on dev #[allow(unused)] fn get_vv(&self, id: ID) -> VersionVector; #[allow(unused)] - fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff; + fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + self.try_find_path(from, to).unwrap() + } + fn try_find_path(&self, from: &Frontiers, to: &Frontiers) -> LoroResult; + #[cfg(test)] fn iter_causal(&self, from: Frontiers, target: IdSpanVector) -> DagCausalIter<'_, Self> where Self: Sized; @@ -135,13 +160,21 @@ impl DagUtils for T { } #[inline] - fn find_meet_and_mode(&self, a_id: &Frontiers, b_id: &Frontiers) -> (MeetAsBase, DiffMode) { - find_meet_and_mode(&|id| self.get(id), a_id, b_id) + fn try_find_meet_and_mode( + &self, + a_id: &Frontiers, + b_id: &Frontiers, + ) -> LoroResult<(MeetAsBase, DiffMode)> { + find_meet_and_mode(&|id| self.try_get(id), a_id, b_id) } #[inline] - fn latest_single_head_critical_version(&self, a_id: &Frontiers, b_id: &Frontiers) -> Frontiers { - latest_single_head_critical_version(&|id| self.get(id), a_id, b_id) + fn try_latest_single_head_critical_version( + &self, + a_id: &Frontiers, + b_id: &Frontiers, + ) -> LoroResult { + latest_single_head_critical_version(&|id| self.try_get(id), a_id, b_id) } #[inline] @@ -149,18 +182,20 @@ impl DagUtils for T { get_version_vector(&|id| self.get(id), id) } - fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + fn try_find_path(&self, from: &Frontiers, to: &Frontiers) -> LoroResult { let mut ans = VersionVectorDiff::default(); if from == to { - return ans; + return Ok(ans); } if from.len() == 1 && to.len() == 1 { let from = from.as_single().unwrap(); let to = to.as_single().unwrap(); if from.peer == to.peer { - let from_span = self.get(from).unwrap(); - let to_span = self.get(to).unwrap(); + let from_span = self + .try_get(from)? + .ok_or(LoroError::FrontiersNotFound(from))?; + let to_span = self.try_get(to)?.ok_or(LoroError::FrontiersNotFound(to))?; if from_span.id_start() == to_span.id_start() { if from.counter < to.counter { ans.forward.insert( @@ -173,7 +208,7 @@ impl DagUtils for T { CounterSpan::new(to.counter + 1, from.counter + 1), ); } - return ans; + return Ok(ans); } if from_span.deps().len() == 1 @@ -183,7 +218,7 @@ impl DagUtils for T { from.peer, CounterSpan::new(to.counter + 1, from.counter + 1), ); - return ans; + return Ok(ans); } if to_span.deps().len() == 1 @@ -193,13 +228,13 @@ impl DagUtils for T { from.peer, CounterSpan::new(from.counter + 1, to.counter + 1), ); - return ans; + return Ok(ans); } } } _walk_to_meet( - &|v| self.get(v), + &|v| self.try_get(v), from, to, &mut |span, node_type| match node_type { @@ -211,9 +246,9 @@ impl DagUtils for T { } }, true, - ); + )?; - ans + Ok(ans) } #[inline(always)] @@ -225,6 +260,7 @@ impl DagUtils for T { } #[inline(always)] + #[cfg(test)] fn iter_causal(&self, from: Frontiers, target: IdSpanVector) -> DagCausalIter<'_, Self> where Self: Sized, @@ -342,19 +378,21 @@ enum NodeType { impl<'a> OrdIdSpan<'a> { #[inline] - fn from_dag_node(id: ID, get: &'a F) -> Option> + fn from_dag_node(id: ID, get: &'a F) -> LoroResult>> where D: DagNode + 'a, - F: Fn(ID) -> Option, + F: Fn(ID) -> LoroResult>, { - let span = get(id)?; + let Some(span) = get(id)? else { + return Ok(None); + }; let span_id = span.id_start(); - Some(OrdIdSpan { + Ok(Some(OrdIdSpan { id: span_id, lamport: span.lamport(), deps: Cow::Owned(span.deps().clone()), len: (id.counter - span_id.counter) as usize + 1, - }) + })) } #[inline] @@ -373,13 +411,13 @@ fn find_meet_and_mode<'a, F, D>( get: &'a F, a_id: &Frontiers, b_id: &Frontiers, -) -> (MeetAsBase, DiffMode) +) -> LoroResult<(MeetAsBase, DiffMode)> where D: DagNode + 'a, - F: Fn(ID) -> Option, + F: Fn(ID) -> LoroResult>, { if b_id.is_empty() { - return (MeetAsBase::Valid(Default::default()), DiffMode::Checkout); + return Ok((MeetAsBase::Valid(Default::default()), DiffMode::Checkout)); } _find_meet_and_mode(get, a_id, b_id) @@ -392,19 +430,19 @@ fn _walk_to_meet<'a, F, D, G>( b_ids: &Frontiers, notify: &mut G, find_path: bool, -) -> FxHashMap +) -> LoroResult> where D: DagNode + 'a, - F: Fn(ID) -> Option, + F: Fn(ID) -> LoroResult>, G: FnMut(IdSpan, NodeType), { let mut ans: FxHashMap = Default::default(); let mut queue: BinaryHeap<(OrdIdSpan, NodeType)> = BinaryHeap::new(); for id in a_ids.iter() { - queue.push((OrdIdSpan::from_dag_node(id, get).unwrap(), NodeType::A)); + queue.push((OrdIdSpan::from_dag_node(id, get)?.unwrap(), NodeType::A)); } for id in b_ids.iter() { - queue.push((OrdIdSpan::from_dag_node(id, get).unwrap(), NodeType::B)); + queue.push((OrdIdSpan::from_dag_node(id, get)?.unwrap(), NodeType::B)); } let mut visited: HashMap = FxHashMap::default(); // invariants in this method: @@ -502,7 +540,7 @@ where } for dep_id in node.deps.as_ref().iter() { - queue.push((OrdIdSpan::from_dag_node(dep_id, get).unwrap(), node_type)); + queue.push((OrdIdSpan::from_dag_node(dep_id, get)?.unwrap(), node_type)); } if node_type != NodeType::Shared { @@ -535,44 +573,46 @@ where } } - ans + Ok(ans) } /// Resolves each frontier id to the span of its containing node, truncated at /// that id. Returns `None` when an id is unavailable (trimmed history). -fn ids_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( +fn ids_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( ids: &Frontiers, get: &'a F, -) -> Option>> { +) -> LoroResult>>> { let mut ans = Vec::with_capacity(ids.len()); for id in ids.iter() { - if let Some(node) = OrdIdSpan::from_dag_node(id, get) { + if let Some(node) = OrdIdSpan::from_dag_node(id, get)? { ans.push(node); } else { - return None; + return Ok(None); } } - Some(ans) + Ok(Some(ans)) } /// The parent set of a span's first op: the node's explicit deps plus the /// implicit same-peer predecessor when it is not already covered by one of them. -fn deps_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( +fn deps_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( node: &OrdIdSpan<'a>, get: &'a F, -) -> Option>> { - let mut deps = ids_to_ord_id_spans(node.deps.as_ref(), get)?; +) -> LoroResult>>> { + let Some(mut deps) = ids_to_ord_id_spans(node.deps.as_ref(), get)? else { + return Ok(None); + }; if node.id.counter > 0 { let prev = node.id.inc(-1); - if let Some(prev) = OrdIdSpan::from_dag_node(prev, get) { + if let Some(prev) = OrdIdSpan::from_dag_node(prev, get)? { if !deps.iter().any(|dep| dep.contains_id(prev.id_last())) { deps.push(prev); } } } - Some(deps) + Ok(Some(deps)) } /// The latest single-head critical version (Eg-walker §3.5/§3.6) of the @@ -590,18 +630,18 @@ fn deps_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( /// endpoint is concurrent with everything below the current level, so no /// later singleton can be valid — bail out to the empty version, which is /// the old behaviour. -fn latest_single_head_critical_version<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( +fn latest_single_head_critical_version<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( get: &'a F, left: &Frontiers, right: &Frontiers, -) -> Frontiers { +) -> LoroResult { let mut queue: BinaryHeap = BinaryHeap::new(); - let Some(spans) = ids_to_ord_id_spans(left, get) else { - return Default::default(); + let Some(spans) = ids_to_ord_id_spans(left, get)? else { + return Ok(Default::default()); }; queue.extend(spans); - let Some(spans) = ids_to_ord_id_spans(right, get) else { - return Default::default(); + let Some(spans) = ids_to_ord_id_spans(right, get)? else { + return Ok(Default::default()); }; queue.extend(spans); @@ -615,7 +655,7 @@ fn latest_single_head_critical_version<'a, D: DagNode + 'a, F: Fn(ID) -> Option< } if queue.is_empty() { - return node.id_last().into(); + return Ok(node.id_last().into()); } if let Some(other) = queue.peek() { @@ -636,17 +676,17 @@ fn latest_single_head_critical_version<'a, D: DagNode + 'a, F: Fn(ID) -> Option< } } - match deps_to_ord_id_spans(&node, get) { + match deps_to_ord_id_spans(&node, get)? { Some(deps) if !deps.is_empty() => { for dep in deps { queue.push(dep); } } - _ => return Default::default(), + _ => return Ok(Default::default()), } } - Default::default() + Ok(Default::default()) } /// Finds the replay base and diff mode for the transition `left -> right`. @@ -671,65 +711,65 @@ fn _find_meet_and_mode<'a, F, D>( get: &'a F, left: &Frontiers, right: &Frontiers, -) -> (MeetAsBase, DiffMode) +) -> LoroResult<(MeetAsBase, DiffMode)> where D: DagNode + 'a, - F: Fn(ID) -> Option, + F: Fn(ID) -> LoroResult>, { if right.is_empty() { - return (MeetAsBase::Valid(Default::default()), DiffMode::Checkout); + return Ok((MeetAsBase::Valid(Default::default()), DiffMode::Checkout)); } if left.is_empty() { if right.len() == 1 { let mut node_id = right.as_single().unwrap(); - let mut node = get(node_id).unwrap(); + let mut node = get(node_id)?.unwrap(); while node.deps().len() == 1 { node_id = node.deps().as_single().unwrap(); - let Some(next) = get(node_id) else { - return ( + let Some(next) = get(node_id)? else { + return Ok(( MeetAsBase::Valid(Default::default()), DiffMode::ImportGreaterUpdates, - ); + )); }; node = next; } if node.deps().is_empty() { - return (MeetAsBase::Valid(Default::default()), DiffMode::Linear); + return Ok((MeetAsBase::Valid(Default::default()), DiffMode::Linear)); } } - return ( + return Ok(( MeetAsBase::Valid(Default::default()), DiffMode::ImportGreaterUpdates, - ); + )); } if left.len() == 1 && right.len() == 1 { let left = left.as_single().unwrap(); let right = right.as_single().unwrap(); if left.peer == right.peer { - let left_span = get(left).unwrap(); - let right_span = get(right).unwrap(); + let left_span = get(left)?.unwrap(); + let right_span = get(right)?.unwrap(); if left_span.id_start() == right_span.id_start() { if left.counter < right.counter { - return (MeetAsBase::Valid(left.into()), DiffMode::Linear); + return Ok((MeetAsBase::Valid(left.into()), DiffMode::Linear)); } else { - return (MeetAsBase::Valid(right.into()), DiffMode::Checkout); + return Ok((MeetAsBase::Valid(right.into()), DiffMode::Checkout)); } } if left_span.deps().len() == 1 && right_span.contains_id(left_span.deps().as_single().unwrap()) { - return (MeetAsBase::Valid(right.into()), DiffMode::Checkout); + return Ok((MeetAsBase::Valid(right.into()), DiffMode::Checkout)); } if right_span.deps().len() == 1 && left_span.contains_id(right_span.deps().as_single().unwrap()) { - return (MeetAsBase::Valid(left.into()), DiffMode::Linear); + return Ok((MeetAsBase::Valid(left.into()), DiffMode::Linear)); } } } @@ -740,21 +780,21 @@ where let mut has_unresolved_unmatched_branch = false; let mut ans: Frontiers = Default::default(); - fn shrink_ancestor_frontiers<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + fn shrink_ancestor_frontiers<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( ids: &Frontiers, get: &'a F, - ) -> Frontiers { + ) -> LoroResult { if ids.len() <= 1 { - return ids.clone(); + return Ok(ids.clone()); } - let mut ids = ids_to_ord_id_spans(ids, get).expect("meet candidates should be in dag"); + let mut ids = ids_to_ord_id_spans(ids, get)?.expect("meet candidates should be in dag"); ids.sort_unstable(); let mut frontiers = Vec::with_capacity(ids.len()); for id in ids.iter().rev() { let mut should_insert = true; for frontier in frontiers.iter().rev() { - if contains_in_ancestors(get, *frontier, id) { + if contains_in_ancestors(get, *frontier, id)? { should_insert = false; break; } @@ -765,19 +805,22 @@ where } } - frontiers.into_iter().collect() + Ok(frontiers.into_iter().collect()) } - fn has_trimmed_history_deps<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + fn has_trimmed_history_deps<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( ids: &Frontiers, get: &'a F, - ) -> bool { - ids.iter().any(|id| { - let Some(node) = OrdIdSpan::from_dag_node(id, get) else { - return true; + ) -> LoroResult { + for id in ids.iter() { + let Some(node) = OrdIdSpan::from_dag_node(id, get)? else { + return Ok(true); }; - ids_to_ord_id_spans(node.deps.as_ref(), get).is_none() - }) + if ids_to_ord_id_spans(node.deps.as_ref(), get)?.is_none() { + return Ok(true); + } + } + Ok(false) } /// Whether every tip in `tips` is contained in the ancestors of `frontiers` @@ -785,21 +828,21 @@ where /// frontier with a shared visited set and prunes by the lowest lamport among /// the still-unproven tips, so the ancestor region is traversed at most once /// per call instead of once per (tip, frontier) pair. - fn all_tips_covered_by_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + fn all_tips_covered_by_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( get: &'a F, frontiers: &Frontiers, tips: &FxHashSet, - ) -> bool { + ) -> LoroResult { let mut remaining = Vec::with_capacity(tips.len()); for id in tips.iter() { - let Some(span) = OrdIdSpan::from_dag_node(*id, get) else { - return false; + let Some(span) = OrdIdSpan::from_dag_node(*id, get)? else { + return Ok(false); }; remaining.push(span); } if remaining.is_empty() { - return true; + return Ok(true); } let mut min_lamport = remaining @@ -810,7 +853,7 @@ where let mut visited = FxHashSet::default(); let mut pending = Vec::new(); for frontier in frontiers.iter() { - if let Some(node) = OrdIdSpan::from_dag_node(frontier, get) { + if let Some(node) = OrdIdSpan::from_dag_node(frontier, get)? { pending.push(node); } } @@ -819,7 +862,7 @@ where let len_before = remaining.len(); remaining.retain(|tip| !node.contains_id(tip.id_last())); if remaining.is_empty() { - return true; + return Ok(true); } if remaining.len() != len_before { @@ -840,14 +883,14 @@ where continue; } - if let Some(deps) = deps_to_ord_id_spans(&node, get) { + if let Some(deps) = deps_to_ord_id_spans(&node, get)? { for dep in deps { pending.push(dep); } } } - false + Ok(false) } /// Verifies the `ImportGreaterUpdates` contract for a multi-head `left`: @@ -866,15 +909,15 @@ where /// through their new parents. The common healthy shape — a change whose /// deps equal the left frontier — passes with a set comparison and no /// graph walk. Trimmed history fails conservatively. - fn new_region_after_all_left_heads<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + fn new_region_after_all_left_heads<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( get: &'a F, left: &Frontiers, right: &Frontiers, - ) -> bool { + ) -> LoroResult { let left_ids: FxHashSet = left.iter().collect(); - let is_old = |id: ID| -> bool { + let is_old = |id: ID| -> LoroResult { if left_ids.contains(&id) { - return true; + return Ok(true); } let mut single = FxHashSet::default(); single.insert(id); @@ -884,11 +927,11 @@ where let mut visited: FxHashSet = FxHashSet::default(); let mut stack: Vec = Vec::new(); for id in right.iter() { - if is_old(id) { + if is_old(id)? { continue; } - let Some(span) = OrdIdSpan::from_dag_node(id, get) else { - return false; + let Some(span) = OrdIdSpan::from_dag_node(id, get)? else { + return Ok(false); }; stack.push(span); } @@ -898,14 +941,14 @@ where continue; } - let Some(parents) = deps_to_ord_id_spans(&span, get) else { - return false; + let Some(parents) = deps_to_ord_id_spans(&span, get)? else { + return Ok(false); }; let mut entry_point = true; let mut old_parents = Frontiers::default(); for parent in parents { let pid = parent.id_last(); - if is_old(pid) { + if is_old(pid)? { old_parents.push(pid); } else { entry_point = false; @@ -918,29 +961,29 @@ where continue; } - if !all_tips_covered_by_ancestors(get, &old_parents, &left_ids) { - return false; + if !all_tips_covered_by_ancestors(get, &old_parents, &left_ids)? { + return Ok(false); } } } - true + Ok(true) } - fn contains_in_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + fn contains_in_ancestors<'a, D: DagNode + 'a, F: Fn(ID) -> LoroResult>>( get: &'a F, frontier: ID, target: &OrdIdSpan<'_>, - ) -> bool { + ) -> LoroResult { let mut visited = FxHashSet::default(); let mut pending = Vec::new(); - let Some(node) = OrdIdSpan::from_dag_node(frontier, get) else { - return false; + let Some(node) = OrdIdSpan::from_dag_node(frontier, get)? else { + return Ok(false); }; pending.push(node); while let Some(node) = pending.pop() { if node.contains_id(target.id_last()) { - return true; + return Ok(true); } if node.lamport_last() < target.lamport_last() { @@ -951,14 +994,14 @@ where continue; } - if let Some(deps) = deps_to_ord_id_spans(&node, get) { + if let Some(deps) = deps_to_ord_id_spans(&node, get)? { for dep in deps { pending.push(dep); } } } - false + Ok(false) } // The third tuple item carries the dependency tips at which this path first @@ -966,7 +1009,7 @@ where // side, those tips tell us whether it was a real concurrent branch or merely // a redundant route into an ancestor we already found. let mut queue: BinaryHeap<(OrdIdSpan, NodeType, Vec)> = BinaryHeap::new(); - for span in ids_to_ord_id_spans(left, get).unwrap() { + for span in ids_to_ord_id_spans(left, get)?.unwrap() { let branch_tips = if left.len() > 1 { vec![span.id_last()] } else { @@ -975,7 +1018,7 @@ where queue.push((span, NodeType::A, branch_tips)); } - for span in ids_to_ord_id_spans(right, get).unwrap() { + for span in ids_to_ord_id_spans(right, get)?.unwrap() { let branch_tips = if right.len() > 1 { vec![span.id_last()] } else { @@ -1049,7 +1092,7 @@ where } } - if let Some(deps) = deps_to_ord_id_spans(&node, get) { + if let Some(deps) = deps_to_ord_id_spans(&node, get)? { if !deps.is_empty() { let starts_new_branches = branch_tips.is_empty() && deps.len() > 1; for dep in deps { @@ -1086,20 +1129,20 @@ where } } - ans = shrink_ancestor_frontiers(&ans, get); + ans = shrink_ancestor_frontiers(&ans, get)?; // A branch can look unmatched merely because a shared node was found first // and was therefore not expanded. In that case another queued path may still // walk into one of the shared node's ancestors. That path is redundant, not a // concurrent branch. Only fall back when an unmatched tip is not causally // covered by the meet candidates we found. let has_uncovered_unmatched_branch = has_unresolved_unmatched_branch - || !all_tips_covered_by_ancestors(get, &ans, &unmatched_branches); + || !all_tips_covered_by_ancestors(get, &ans, &unmatched_branches)?; // A genuine concurrent branch invalidates the meet as a replay base. The // caller has to retreat to a critical version; which one is its choice, so // it can try a cheap multi-head cut before paying for the full-DAG // single-head descent. let mut needs_critical_retreat = - has_uncovered_unmatched_branch && !has_trimmed_history_deps(&ans, get); + has_uncovered_unmatched_branch && !has_trimmed_history_deps(&ans, get)?; if has_uncovered_unmatched_branch { is_right_greater = false; @@ -1110,7 +1153,7 @@ where is_right_greater = true; } - if is_right_greater && left.len() > 1 && !new_region_after_all_left_heads(get, left, right) { + if is_right_greater && left.len() > 1 && !new_region_after_all_left_heads(get, left, right)? { // Version inclusion holds, but part of the new region entered old // history through a strict subset of the left heads and is therefore // concurrent with the others. `ImportGreaterUpdates` promises "no @@ -1144,7 +1187,7 @@ where } else { MeetAsBase::Valid(ans) }; - (meet, mode) + Ok((meet, mode)) } pub fn remove_included_frontiers(frontiers: &mut VersionVector, new_change_deps: &[ID]) { @@ -1745,13 +1788,7 @@ mod tests { let root = node(1, 0, 1, 0, Frontiers::default()); let x = node(2, 0, 1, 1, ID::new(1, 0).into()); let y = node(3, 0, 1, 1, ID::new(1, 0).into()); - let merge = node( - 4, - 0, - 1, - 2, - Frontiers::from([ID::new(2, 0), ID::new(3, 0)]), - ); + let merge = node(4, 0, 1, 2, Frontiers::from([ID::new(2, 0), ID::new(3, 0)])); let dag = TestDag::new(vec![root, x, y, merge], ID::new(4, 0).into()); let left = Frontiers::from([ID::new(2, 0), ID::new(3, 0)]); diff --git a/crates/loro-internal/src/dag/iter.rs b/crates/loro-internal/src/dag/iter.rs index 4c824b5b8..8427b1fde 100644 --- a/crates/loro-internal/src/dag/iter.rs +++ b/crates/loro-internal/src/dag/iter.rs @@ -197,7 +197,12 @@ pub(crate) struct IterReturn { } impl<'a, T: DagNode, D: Dag + Debug> DagCausalIter<'a, D> { + #[cfg(test)] pub fn new(dag: &'a D, from: Frontiers, target: IdSpanVector) -> Self { + Self::try_new(dag, from, target).unwrap() + } + + pub fn try_new(dag: &'a D, from: Frontiers, target: IdSpanVector) -> LoroResult { let mut out_degrees: FxHashMap = FxHashMap::default(); let mut succ: BTreeMap> = BTreeMap::default(); let mut stack = Vec::new(); @@ -213,7 +218,7 @@ impl<'a, T: DagNode, D: Dag + Debug> DagCausalIter<'a, D> { // if out_degree is 0, then it can be iterated directly while let Some(id) = q.pop() { let client = id.peer; - let node = dag.get(id).unwrap(); + let node = dag.try_get(id)?.ok_or(LoroError::FrontiersNotFound(id))?; let deps = node.deps(); out_degrees.insert( id, @@ -273,14 +278,14 @@ impl<'a, T: DagNode, D: Dag + Debug> DagCausalIter<'a, D> { true }); - Self { + Ok(Self { dag, frontier: from, target, out_degrees, succ, stack, - } + }) } } @@ -314,6 +319,7 @@ impl> Iterator for DagCausalIter<'_, D> { } // node_id may point into the middle of the node, we need to slice + // try_new has loaded every target node before iteration starts. let node = self.dag.get(node_id).unwrap(); // node start_id may be smaller than node_id let counter = node.id_span().counter; diff --git a/crates/loro-internal/src/diff_calc.rs b/crates/loro-internal/src/diff_calc.rs index 825ddcf19..07e6591b8 100644 --- a/crates/loro-internal/src/diff_calc.rs +++ b/crates/loro-internal/src/diff_calc.rs @@ -122,9 +122,9 @@ fn changed_containers_between( oplog: &OpLog, before: &VersionVector, after: &VersionVector, -) -> FxHashSet { +) -> loro_common::LoroResult> { let (retreat, forward) = before.diff_iter(after); - oplog.containers_in_spans(retreat.chain(forward)) + oplog.try_containers_in_spans(retreat.chain(forward)) } impl DiffCalculator { @@ -160,6 +160,26 @@ impl DiffCalculator { /// It's the expected diff mode inferred from the two version, which can reflect the direction of the /// change). pub(crate) fn calc_diff_internal( + &mut self, + oplog: &OpLog, + before: &VersionVector, + before_frontiers: &Frontiers, + after: &VersionVector, + after_frontiers: &Frontiers, + container_filter: Option<&dyn Fn(ContainerIdx) -> bool>, + ) -> (Vec, DiffMode) { + self.try_calc_diff_internal( + oplog, + before, + before_frontiers, + after, + after_frontiers, + container_filter, + ) + .unwrap() + } + + pub(crate) fn try_calc_diff_internal( &mut self, oplog: &super::oplog::OpLog, before: &crate::VersionVector, @@ -167,9 +187,9 @@ impl DiffCalculator { after: &crate::VersionVector, after_frontiers: &Frontiers, container_filter: Option<&dyn Fn(ContainerIdx) -> bool>, - ) -> (Vec, DiffMode) { + ) -> loro_common::LoroResult<(Vec, DiffMode)> { if before == after { - return (Vec::new(), DiffMode::Linear); + return Ok((Vec::new(), DiffMode::Linear)); } let s = tracing::span!(tracing::Level::INFO, "DiffCalc", ?before, ?after,); @@ -185,14 +205,20 @@ impl DiffCalculator { concurrent_containers, }, iter, - ) = oplog.iter_from_replay_base_causally(before, before_frontiers, after, after_frontiers); + ) = oplog.try_iter_from_replay_base_causally( + before, + before_frontiers, + after, + after_frontiers, + )?; // A conservative replay base may be much older than `before`. The causal replay // still needs that common history as position context, but containers // whose ops are present on both sides cannot contribute to the diff. // Without this filter, every such List/Text/MovableList can trigger its // own full-history safety rebuild below. - let changed_containers = - (&replay_base != before).then(|| changed_containers_between(oplog, before, after)); + let changed_containers = (&replay_base != before) + .then(|| changed_containers_between(oplog, before, after)) + .transpose()?; // Two distinct mode values live in this function — do not conflate them: // - `origin_diff_mode` describes the DIRECTION of the transition // (Checkout can go backwards; the other modes imply `after ⊇ before`). @@ -219,7 +245,8 @@ impl DiffCalculator { let affected_set = { loro_common::debug!("replay_base: {:?} mode={:?}", &replay_base, calc_mode); let mut started_set = FxHashSet::default(); - for (change, (start_counter, end_counter), vv) in iter { + for item in iter { + let (change, (start_counter, end_counter), vv) = item?; let iter_start = change .ops .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) @@ -430,10 +457,10 @@ impl DiffCalculator { } } - ( + Ok(( ans.into_values().map(|x| x.1).collect_vec(), origin_diff_mode, - ) + )) } // TODO: we may remove depth info @@ -2340,7 +2367,7 @@ fn causal_existing_peer_import_uses_current_version_as_replay_base() { assert_eq!(base.vv, before); assert_eq!(base.diff_mode, DiffMode::ImportGreaterUpdates); assert_eq!( - changed_containers_between(&oplog, &before, &after), + changed_containers_between(&oplog, &before, &after).unwrap(), [expected_idx].into_iter().collect() ); @@ -2398,7 +2425,7 @@ fn conservative_replay_only_builds_calculators_for_changed_containers() { ); assert_eq!(base.diff_mode, DiffMode::Import); assert_eq!( - changed_containers_between(&oplog, &before, &after), + changed_containers_between(&oplog, &before, &after).unwrap(), [expected_idx, list_idx].into_iter().collect() ); @@ -2461,7 +2488,7 @@ fn conservative_checkout_replay_filters_to_retreat_changed_containers() { ); assert_eq!(base.diff_mode, DiffMode::Checkout); assert_eq!( - changed_containers_between(&oplog, &before, &after), + changed_containers_between(&oplog, &before, &after).unwrap(), [expected_idx].into_iter().collect() ); diff --git a/crates/loro-internal/src/encoding.rs b/crates/loro-internal/src/encoding.rs index a6a53e960..390eae05f 100644 --- a/crates/loro-internal/src/encoding.rs +++ b/crates/loro-internal/src/encoding.rs @@ -235,7 +235,7 @@ pub(crate) fn decode_oplog( parsed: ParsedHeaderAndBody, ) -> Result { let changes = decode_oplog_changes(oplog, parsed)?; - let result = apply_decoded_changes_to_oplog(oplog, changes); + let result = apply_decoded_changes_to_oplog(oplog, changes)?; if result.has_deps_before_shallow_root { return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); } @@ -266,13 +266,13 @@ pub(crate) struct ApplyDecodedChangesResult { pub(crate) fn apply_decoded_changes_to_oplog( oplog: &mut OpLog, changes: Vec, -) -> ApplyDecodedChangesResult { +) -> LoroResult { let ImportChangesResult { mut imported, latest_ids, pending_changes, changes_that_have_deps_before_shallow_root, - } = import_changes_to_oplog(changes, oplog); + } = import_changes_to_oplog(changes, oplog)?; // TODO: PERF: should we use hashmap to filter latest_ids with the same peer first? oplog.try_apply_pending(latest_ids, Some(&mut imported)); @@ -281,13 +281,13 @@ pub(crate) fn apply_decoded_changes_to_oplog( // remain in the returned pending range. let pending = oplog.import_unknown_lamport_pending_changes(pending_changes, Some(&mut imported)); - ApplyDecodedChangesResult { + Ok(ApplyDecodedChangesResult { status: ImportStatus { success: imported, pending: (!pending.is_empty()).then_some(pending), }, has_deps_before_shallow_root: !changes_that_have_deps_before_shallow_root.is_empty(), - } + }) } pub(crate) struct ParsedHeaderAndBody<'a> { diff --git a/crates/loro-internal/src/encoding/fast_snapshot.rs b/crates/loro-internal/src/encoding/fast_snapshot.rs index 08ed522b6..0bf0f5130 100644 --- a/crates/loro-internal/src/encoding/fast_snapshot.rs +++ b/crates/loro-internal/src/encoding/fast_snapshot.rs @@ -275,7 +275,7 @@ pub(crate) fn decode_snapshot_inner( impl OpLog { pub(super) fn decode_change_store(&mut self, bytes: bytes::Bytes) -> LoroResult<()> { let v = self.change_store().import_all(bytes)?; - self.dag.set_version_by_fast_snapshot_import(v); + self.dag.set_version_by_fast_snapshot_import(v)?; self.refresh_visible_op_count(); Ok(()) } diff --git a/crates/loro-internal/src/encoding/json_schema.rs b/crates/loro-internal/src/encoding/json_schema.rs index 80fea8d8e..055a87c51 100644 --- a/crates/loro-internal/src/encoding/json_schema.rs +++ b/crates/loro-internal/src/encoding/json_schema.rs @@ -41,35 +41,37 @@ fn refine_vv(vv: &VersionVector, oplog: &OpLog) -> VersionVector { refined } -pub(crate) fn export_json<'a, 'c: 'a>( +pub(crate) fn try_export_json<'a, 'c: 'a>( oplog: &'c OpLog, start_vv: &VersionVector, end_vv: &VersionVector, with_peer_compression: bool, -) -> JsonSchema { +) -> LoroResult { let actual_start_vv = refine_vv(start_vv, oplog); let actual_end_vv = refine_vv(end_vv, oplog); - let frontiers = oplog.dag.vv_to_frontiers(&actual_start_vv); + let frontiers = oplog.dag.try_vv_to_frontiers(&actual_start_vv)?; let diff_changes = init_encode(oplog, &actual_start_vv, &actual_end_vv); + // Legacy iteration records a bad block and skips it. Do not return that gap. + oplog.check_history_parsable()?; if with_peer_compression { let mut peer_register = ValueRegister::::new(); let changes = encode_changes(&diff_changes, &oplog.arena, Some(&mut peer_register)); - JsonSchema { + Ok(JsonSchema { changes, schema_version: SCHEMA_VERSION, peers: Some(peer_register.unwrap_vec()), start_version: frontiers, - } + }) } else { let changes = encode_changes(&diff_changes, &oplog.arena, None); - JsonSchema { + Ok(JsonSchema { changes, schema_version: SCHEMA_VERSION, peers: None, start_version: frontiers, - } + }) } } @@ -1899,17 +1901,21 @@ mod tests { list.insert(0, "a").unwrap(); list.insert(0, "b").unwrap(); list.insert(0, "c").unwrap(); - let json = doc.export_json_updates( - &VersionVector::from_iter(vec![(0, 1)]), - &VersionVector::from_iter(vec![(0, 2)]), - true, - ); + let json = doc + .export_json_updates( + &VersionVector::from_iter(vec![(0, 1)]), + &VersionVector::from_iter(vec![(0, 2)]), + true, + ) + .unwrap(); assert_eq!(json.changes[0].ops.len(), 1); - let json = doc.export_json_updates( - &VersionVector::from_iter(vec![(0, 0)]), - &VersionVector::from_iter(vec![(0, 2)]), - true, - ); + let json = doc + .export_json_updates( + &VersionVector::from_iter(vec![(0, 0)]), + &VersionVector::from_iter(vec![(0, 2)]), + true, + ) + .unwrap(); assert_eq!(json.changes[0].ops.len(), 2); } } diff --git a/crates/loro-internal/src/encoding/outdated_encode_reordered.rs b/crates/loro-internal/src/encoding/outdated_encode_reordered.rs index 4a4ba9b9e..14720d3de 100644 --- a/crates/loro-internal/src/encoding/outdated_encode_reordered.rs +++ b/crates/loro-internal/src/encoding/outdated_encode_reordered.rs @@ -40,7 +40,7 @@ pub(crate) struct ImportChangesResult { pub(crate) fn import_changes_to_oplog( changes: Vec, oplog: &mut OpLog, -) -> ImportChangesResult { +) -> LoroResult { let mut pending_changes = Vec::new(); let mut latest_ids = Vec::new(); let mut changes_before_shallow_root = Vec::new(); @@ -51,14 +51,17 @@ pub(crate) fn import_changes_to_oplog( continue; } - if oplog.dag.import_deps_before_shallow_root(&change.deps) { + if oplog + .dag + .try_import_deps_before_shallow_root(&change.deps)? + { changes_before_shallow_root.push(change); continue; } latest_ids.push(change.id_last()); // calc lamport or pending if its deps are not satisfied - match oplog.dag.get_change_lamport_from_deps(&change.deps) { + match oplog.dag.try_get_change_lamport_from_deps(&change.deps)? { Some(lamport) => change.lamport = lamport, None => { pending_changes.push(change); @@ -74,12 +77,12 @@ pub(crate) fn import_changes_to_oplog( oplog.insert_new_change(change, false); } - ImportChangesResult { + Ok(ImportChangesResult { latest_ids, pending_changes, changes_that_have_deps_before_shallow_root: changes_before_shallow_root, imported, - } + }) } mod encode { diff --git a/crates/loro-internal/src/encoding/shallow_snapshot.rs b/crates/loro-internal/src/encoding/shallow_snapshot.rs index 074c60fd3..6162ade77 100644 --- a/crates/loro-internal/src/encoding/shallow_snapshot.rs +++ b/crates/loro-internal/src/encoding/shallow_snapshot.rs @@ -307,7 +307,7 @@ pub(crate) fn export_shallow_snapshot_inner( cached_root: CachedShallowRoot, ) -> Result<(Snapshot, Frontiers), LoroEncodeError> { let oplog = doc.oplog().lock(); - let start_from = calc_shallow_doc_start(&oplog, start_from, oplog.frontiers()); + let start_from = calc_shallow_doc_start(&oplog, start_from, oplog.frontiers())?; // `root_vv` is the version of the state at the shallow root; `start_vv` // additionally excludes the frontier ops themselves because the retained // history must include them. @@ -324,14 +324,14 @@ pub(crate) fn export_shallow_snapshot_inner( if !start_from.is_empty() { assert!(start_from.len() == 1); let id = start_from.as_single().unwrap(); - let node = oplog.dag.get(id).unwrap(); + let node = oplog.dag.try_get(id)?.unwrap(); if id.counter == node.cnt { - let vv = oplog.dag().frontiers_to_vv(&node.deps).unwrap(); + let vv = oplog.dag().try_frontiers_to_vv(&node.deps)?.unwrap(); assert_eq!(vv, start_vv); } else { let vv = oplog .dag() - .frontiers_to_vv(&Frontiers::from(id.inc(-1))) + .try_frontiers_to_vv(&Frontiers::from(id.inc(-1)))? .unwrap(); assert_eq!(vv, start_vv); } @@ -820,7 +820,7 @@ pub(crate) fn export_state_only_snapshot( w: &mut W, ) -> Result { let oplog = doc.oplog().lock(); - let start_from = calc_shallow_doc_start(&oplog, target_frontiers, target_frontiers); + let start_from = calc_shallow_doc_start(&oplog, target_frontiers, target_frontiers)?; let root_vv = frontiers_to_vv_for_export(&oplog, &start_from, "export_state_only_snapshot")?; let mut start_vv = root_vv.clone(); for id in start_from.iter() { @@ -899,7 +899,7 @@ fn frontiers_to_vv_for_export( frontiers: &Frontiers, context: &str, ) -> Result { - oplog.dag().frontiers_to_vv(frontiers).ok_or_else(|| { + oplog.dag().try_frontiers_to_vv(frontiers)?.ok_or_else(|| { LoroEncodeError::FrontiersNotFound(format!( "{context}: unreachable frontiers {frontiers:?}" )) @@ -941,7 +941,7 @@ fn calc_shallow_doc_start( oplog: &crate::OpLog, frontiers: &Frontiers, retained_to: &Frontiers, -) -> Frontiers { +) -> Result { if frontiers.is_empty() { return clamp_to_shallow_root(oplog, Frontiers::default()); } @@ -950,7 +950,7 @@ fn calc_shallow_doc_start( // version exists, e.g. for independent heads with no common history. let root = oplog .dag() - .latest_single_head_critical_version(frontiers, retained_to); + .try_latest_single_head_critical_version(frontiers, retained_to)?; let mut ans = Frontiers::new(); for id in root.iter() { @@ -979,19 +979,22 @@ fn calc_shallow_doc_start( clamp_to_shallow_root(oplog, ans) } -fn clamp_to_shallow_root(oplog: &crate::OpLog, frontiers: Frontiers) -> Frontiers { +fn clamp_to_shallow_root( + oplog: &crate::OpLog, + frontiers: Frontiers, +) -> Result { if oplog.shallow_since_vv().is_empty() { - return frontiers; + return Ok(frontiers); } - let Some(vv) = oplog.dag().frontiers_to_vv(&frontiers) else { - return oplog.shallow_since_frontiers().clone(); + let Some(vv) = oplog.dag().try_frontiers_to_vv(&frontiers)? else { + return Ok(oplog.shallow_since_frontiers().clone()); }; if vv.includes_vv(&oplog.shallow_since_vv().to_vv()) { - frontiers + Ok(frontiers) } else { - oplog.shallow_since_frontiers().clone() + Ok(oplog.shallow_since_frontiers().clone()) } } @@ -1076,7 +1079,11 @@ pub(crate) fn encode_snapshot_at( "encode_snapshot_at: state is unexpectedly still in a transaction", )); } - let Some(oplog_bytes) = oplog.fork_changes_up_to(frontiers) else { + let oplog_bytes = match oplog.fork_changes_up_to(frontiers) { + Ok(bytes) => bytes, + Err(err) => break 'block Err(err.into()), + }; + let Some(oplog_bytes) = oplog_bytes else { break 'block Err(LoroEncodeError::FrontiersNotFound(format!( "frontiers: {:?} when export in SnapshotAt mode", frontiers @@ -1096,7 +1103,11 @@ pub(crate) fn encode_snapshot_at( } } - let Some(version) = oplog.dag.frontiers_to_vv(frontiers) else { + let version = match oplog.dag.try_frontiers_to_vv(frontiers) { + Ok(version) => version, + Err(err) => break 'block Err(err.into()), + }; + let Some(version) = version else { break 'block Err(LoroEncodeError::FrontiersNotFound(format!( "frontiers: {:?} when export in SnapshotAt mode", frontiers diff --git a/crates/loro-internal/src/loro.rs b/crates/loro-internal/src/loro.rs index a92afd381..917494531 100644 --- a/crates/loro-internal/src/loro.rs +++ b/crates/loro-internal/src/loro.rs @@ -32,7 +32,7 @@ use crate::{ subscription::{LocalUpdateCallback, Observer, Subscriber}, undo::DiffBatch, utils::subscription::{SubscriberSetWithQueue, Subscription}, - version::{shrink_frontiers, Frontiers, ImVersionVector, VersionRange, VersionVectorDiff}, + version::{Frontiers, ImVersionVector, VersionRange, VersionVectorDiff}, ChangeMeta, DocDiff, HandlerTrait, InternalString, ListHandler, LoroDoc, LoroError, MapHandler, VersionVector, }; @@ -707,7 +707,7 @@ impl LoroDoc { let result = f(&mut oplog); if &old_vv != oplog.vv() { let mut diff = DiffCalculator::new(false); - let (diff, diff_mode) = diff.calc_diff_internal( + let computed = diff.try_calc_diff_internal( &oplog, &old_vv, &old_frontiers, @@ -715,6 +715,13 @@ impl LoroDoc { oplog.dag.get_frontiers(), None, ); + let (diff, diff_mode) = match computed { + Ok(diff) => diff, + Err(err) => { + oplog.rollback_owned_import(owns_rollback, &mut self.state.lock()); + return Err(err); + } + }; let mut state = self.state.lock(); if let Err(e) = state.apply_diff( InternalDocDiff { @@ -785,7 +792,13 @@ impl LoroDoc { // Read once: attach/detach can change the flag without the oplog lock. // The apply branch must use the same mode as preflight's rollback decision. let detached = self.is_detached(); - let preflight = oplog.preflight_import_changes(&changes, detached); + let preflight = match oplog.preflight_import_changes(&changes, detached) { + Ok(preflight) => preflight, + Err(err) => { + oplog.rollback_arena(arena_checkpoint, &mut self.state.lock()); + return Err(err); + } + }; if preflight.has_deps_before_shallow_root && (detached || !preflight.applies_to_dag) { oplog.rollback_arena(arena_checkpoint, &mut self.state.lock()); return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); @@ -799,7 +812,15 @@ impl LoroDoc { if owns_rollback { oplog.begin_import_rollback_with_arena(arena_checkpoint); } - let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + let result = match encoding::apply_decoded_changes_to_oplog(&mut oplog, changes) { + Ok(result) => result, + Err(err) => { + if owns_rollback { + oplog.rollback_import(&mut self.state.lock()); + } + return Err(err); + } + }; if owns_rollback { if let Err(e) = oplog.validate_movable_list_elem_refs_in_import_scope() { oplog.rollback_import(&mut self.state.lock()); @@ -815,8 +836,23 @@ impl LoroDoc { } if !preflight.applies_to_dag { - let pending_root_containers = pending_root_containers_to_materialize(&oplog, &changes); - let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + let pending_root_containers = + match pending_root_containers_to_materialize(&oplog, &changes) { + Ok(ids) => ids, + Err(err) => { + oplog.rollback_arena(arena_checkpoint, &mut self.state.lock()); + return Err(err); + } + }; + let result = match encoding::apply_decoded_changes_to_oplog(&mut oplog, changes) { + Ok(result) => result, + Err(err) => { + if !oplog.has_import_rollback() { + oplog.rollback_arena(arena_checkpoint, &mut self.state.lock()); + } + return Err(err); + } + }; if result.has_deps_before_shallow_root { oplog.rollback_arena(arena_checkpoint, &mut self.state.lock()); return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); @@ -832,7 +868,7 @@ impl LoroDoc { return Ok(result.status); } - let old_vv = oplog.vv().clone(); + let old_vv = Arc::new(oplog.vv().clone()); let old_frontiers = oplog.frontiers().clone(); // Checked before the changes are applied, while the store still holds only old history. let isolated_batch = isolated_scalar_root_batch(&oplog, &changes).filter(|(_, names)| { @@ -842,13 +878,20 @@ impl LoroDoc { !state.store.contains_id(&id) }) }); - let rollback_enabled = preflight.needs_state_apply_rollback; + let rollback_enabled = + preflight.needs_state_apply_rollback || oplog.change_store().may_have_unparsed_bodies(); if rollback_enabled { - oplog.begin_import_rollback_with_arena(arena_checkpoint); + oplog.begin_import_rollback_with_version(arena_checkpoint, old_vv.clone()); } - let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); - if &old_vv != oplog.vv() { + let result = match encoding::apply_decoded_changes_to_oplog(&mut oplog, changes) { + Ok(result) => result, + Err(err) => { + oplog.rollback_owned_import(rollback_enabled, &mut self.state.lock()); + return Err(err); + } + }; + if old_vv.as_ref() != oplog.vv() { // The preflight enables rollback whenever the imported or unlocked // pending changes hold movable-list ops, so other imports skip the scan. if rollback_enabled { @@ -870,37 +913,54 @@ impl LoroDoc { *end == old_end.max(component_end) }) }); - let (diff, diff_mode) = if let Some((component_vv, _)) = isolated_batch { - #[cfg(test)] - ISOLATED_FAST_PATH_HITS.with(|hits| hits.set(hits.get() + 1)); - let component_frontiers = oplog.dag.vv_to_frontiers(&component_vv); - let (diff, _) = calc.calc_diff_internal( - &oplog, - &VersionVector::default(), - &Frontiers::default(), - &component_vv, - &component_frontiers, - None, - ); - (diff, DiffMode::Import) - } else { - calc.calc_diff_internal( - &oplog, - &old_vv, - &old_frontiers, - oplog.vv(), - oplog.dag.get_frontiers(), - None, - ) + let computed = (|| -> LoroResult<_> { + let (diff, diff_mode) = if let Some((component_vv, _)) = isolated_batch { + #[cfg(test)] + ISOLATED_FAST_PATH_HITS.with(|hits| hits.set(hits.get() + 1)); + let component_frontiers = oplog.dag.try_vv_to_frontiers(&component_vv)?; + let (diff, _) = calc.try_calc_diff_internal( + &oplog, + &VersionVector::default(), + &Frontiers::default(), + &component_vv, + &component_frontiers, + None, + )?; + (diff, DiffMode::Import) + } else { + calc.try_calc_diff_internal( + &oplog, + &old_vv, + &old_frontiers, + oplog.vv(), + oplog.dag.get_frontiers(), + None, + )? + }; + oplog.check_history_parsable()?; + Ok((diff, diff_mode)) + })(); + let (diff, diff_mode) = match computed { + Ok(diff) => diff, + Err(err) => { + oplog.rollback_owned_import(rollback_enabled, &mut self.state.lock()); + return Err(err); + } }; let mut state = self.state.lock(); - let diff = recalc_in_checkout_mode_if_needed( + let diff = match recalc_in_checkout_mode_if_needed( &mut state, &oplog, &old_vv, &old_frontiers, diff, - ); + ) { + Ok(diff) => diff, + Err(err) => { + oplog.rollback_owned_import(rollback_enabled, &mut state); + return Err(err); + } + }; if let Err(e) = state.apply_diff( InternalDocDiff { origin, @@ -971,7 +1031,16 @@ impl LoroDoc { start_vv: &VersionVector, end_vv: &VersionVector, with_peer_compression: bool, - ) -> JsonSchema { + ) -> LoroResult { + self.try_export_json_updates(start_vv, end_vv, with_peer_compression) + } + + pub fn try_export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + with_peer_compression: bool, + ) -> LoroResult { self.with_barrier(|| { let oplog = self.oplog.lock(); let mut start_vv = start_vv; @@ -995,7 +1064,7 @@ impl LoroDoc { } } - crate::encoding::json_schema::export_json( + crate::encoding::json_schema::try_export_json( &oplog, start_vv, end_vv, @@ -1020,12 +1089,29 @@ impl LoroDoc { self.oplog.lock().vv().clone() } - /// Get the version vector of the current [DocState] + /// Get the version vector of the current [DocState]. + /// + /// When the state frontiers are the op-log frontiers, this clones the cached + /// op-log version vector and does not parse change blocks. Otherwise it + /// walks the frontiers. #[inline] - pub fn state_vv(&self) -> VersionVector { + pub fn state_vv(&self) -> LoroResult { + self.try_state_vv() + } + + /// [`state_vv`](Self::state_vv), including a decode error from unreadable history. + /// + /// `Ok(None)` from the frontier walk (the frontiers are not in the dag) is + /// [`LoroError::FrontiersNotFound`] for an id that can be named, not an empty + /// version vector. + pub fn try_state_vv(&self) -> LoroResult { let oplog = self.oplog.lock(); - let f = &self.state.lock().frontiers; - oplog.dag.frontiers_to_vv(f).unwrap() + let frontiers = self.state.lock().frontiers.clone(); + if &frontiers == oplog.frontiers() { + // `OpLog::vv` is the cached DAG version vector. It does not parse blocks. + return Ok(oplog.vv().clone()); + } + frontiers_to_vv_or_not_found(&oplog, &frontiers) } pub fn get_by_path(&self, path: &[Index]) -> Option { @@ -1495,6 +1581,14 @@ impl LoroDoc { return Err(LoroError::EditWhenDetached); } + // Rollback checks the state out through the current frontiers. If that + // history cannot be read, refuse before the batch mutates anything. + { + let oplog = self.oplog.lock(); + let frontiers = self.state.lock().frontiers.clone(); + frontiers_to_vv_or_not_found(&oplog, &frontiers)?; + } + let scope = self.begin_apply_diff_scope(); let result = self._apply_diff( diff, @@ -1505,8 +1599,8 @@ impl LoroDoc { if let Some(scope) = scope { if result.is_ok() { self.end_apply_diff_scope(scope); - } else { - self.rollback_apply_diff_scope(scope); + } else if let Err(err) = self.rollback_apply_diff_scope(scope) { + return Err(err); } } result @@ -1552,7 +1646,7 @@ impl LoroDoc { /// rollback scope, so that the diff calculator can compute the checkout from the batch's /// last op back to the version before the batch; then the scope is rolled back and the /// ops before the batch are put back into the DAG's version as pending local ops. - fn rollback_apply_diff_scope(&self, scope: u64) { + fn rollback_apply_diff_scope(&self, scope: u64) -> LoroResult<()> { let mut guard = self.txn.lock(); let Some(txn) = guard .as_mut() @@ -1565,7 +1659,7 @@ impl LoroDoc { txn.end_rollback_scope(scope); } self.state.lock().end_local_rollback(scope); - return; + return Ok(()); }; let batch = txn.roll_back_scope(scope); @@ -1614,39 +1708,35 @@ impl LoroDoc { batch_last, Frontiers::from_id(ID::new(peer, batch.batch_end - 1)) ); - let from = oplog - .dag - .frontiers_to_vv(&batch_last) - .expect("the batch's ops are in the DAG"); - let to = oplog - .dag - .frontiers_to_vv(&before_batch) - .expect("the version before the batch is in the DAG"); - let (diff, diff_mode) = DiffCalculator::new(false).calc_diff_internal( - &oplog, - &from, - &batch_last, - &to, - &before_batch, - None, - ); - // `DocState::apply_diff` refuses to run inside a transaction, and records nothing - // without its event recorder. - state.abort_txn(); - let recorder = state.take_event_recorder(); - state - .apply_diff( + let undone = (|| -> LoroResult<()> { + let from = frontiers_to_vv_or_not_found(&oplog, &batch_last)?; + let to = frontiers_to_vv_or_not_found(&oplog, &before_batch)?; + let (diff, diff_mode) = DiffCalculator::new(false).try_calc_diff_internal( + &oplog, + &from, + &batch_last, + &to, + &before_batch, + None, + )?; + // `DocState::apply_diff` refuses to run inside a transaction, and records nothing + // without its event recorder. + state.abort_txn(); + let recorder = state.take_event_recorder(); + let applied = state.apply_diff( InternalDocDiff { origin: Default::default(), by: EventTriggerKind::Checkout, diff: Cow::Owned(diff), - new_version: Cow::Owned(before_batch), + new_version: Cow::Owned(before_batch.clone()), }, diff_mode, - ) - .expect("undoing the ops of a failed apply_diff from the state"); - state.restore_event_recorder(recorder); - state.resume_txn(); + ); + state.restore_event_recorder(recorder); + state.resume_txn(); + applied + })(); + // The temporary change must not stay in the op log if the checkout cannot be read. oplog.rollback_import_keeping_arena(&state); if txn_ops_before > 0 { oplog.dag.update_version_on_new_local_op( @@ -1657,8 +1747,10 @@ impl LoroDoc { ); oplog.refresh_visible_op_count(); } + undone?; } state.finish_local_rollback(scope, batch.peer, batch.batch_start); + Ok(()) } /// Apply a diff to the current state. @@ -2207,7 +2299,11 @@ impl LoroDoc { a: &Frontiers, b: &Frontiers, ) -> Result, FrontiersNotIncluded> { - self.oplog().lock().cmp_frontiers(a, b) + let oplog = self.oplog.lock(); + oplog + .check_history_parsable() + .map_err(FrontiersNotIncluded::unreadable)?; + oplog.cmp_frontiers(a, b) } pub fn subscribe_root(&self, callback: Subscriber) -> Subscription { @@ -2536,7 +2632,7 @@ impl LoroDoc { } let frontiers = if to_shrink_frontiers { - shrink_frontiers(frontiers, &oplog.dag).map_err(LoroError::FrontiersNotFound)? + crate::version::try_shrink_frontiers(frontiers, &oplog.dag)? } else { frontiers.clone() }; @@ -2553,24 +2649,33 @@ impl LoroDoc { } } - let before = oplog.dag.frontiers_to_vv(&state.frontiers).ok_or_else(|| { - LoroError::NotFoundError( - format!( - "Cannot find the current state version {:?}", - state.frontiers + let before = oplog + .dag + .try_frontiers_to_vv(&state.frontiers)? + .ok_or_else(|| { + LoroError::NotFoundError( + format!( + "Cannot find the current state version {:?}", + state.frontiers + ) + .into_boxed_str(), ) - .into_boxed_str(), - ) - })?; - let Some(after) = &oplog.dag.frontiers_to_vv(&frontiers) else { + })?; + let Some(after) = &oplog.dag.try_frontiers_to_vv(&frontiers)? else { return Err(LoroError::NotFoundError( format!("Cannot find the specified version {:?}", frontiers).into_boxed_str(), )); }; + let (diff, diff_mode) = calc.try_calc_diff_internal( + &oplog, + &before, + &state.frontiers, + after, + &frontiers, + None, + )?; self.set_detached(true); - let (diff, diff_mode) = - calc.calc_diff_internal(&oplog, &before, &state.frontiers, after, &frontiers, None); state.apply_diff( InternalDocDiff { origin, @@ -2600,6 +2705,34 @@ impl LoroDoc { self.oplog.lock().dag.frontiers_to_vv(frontiers) } + /// Converts frontiers to a version vector, reporting unreadable history. + pub fn try_frontiers_to_vv(&self, frontiers: &Frontiers) -> LoroResult> { + let oplog = self.oplog.lock(); + oplog.check_history_parsable()?; + oplog.dag.try_frontiers_to_vv(frontiers) + } + + /// Converts a version vector to frontiers, reporting unreadable history. + pub fn try_vv_to_frontiers(&self, vv: &VersionVector) -> LoroResult { + let oplog = self.oplog.lock(); + oplog.check_history_parsable()?; + oplog.dag.try_vv_to_frontiers(vv) + } + + /// Removes redundant frontiers, reporting unreadable history or an absent id. + pub fn try_minimize_frontiers(&self, frontiers: &Frontiers) -> LoroResult { + let oplog = self.oplog.lock(); + oplog.check_history_parsable()?; + crate::version::try_shrink_frontiers(frontiers, &oplog.dag) + } + + /// Reads change metadata, distinguishing unreadable history from an absent id. + pub fn try_get_change(&self, id: ID) -> LoroResult> { + let oplog = self.oplog.lock(); + let change = oplog.try_get_change_at(id)?; + Ok(change.as_deref().map(ChangeMeta::from_change)) + } + /// Import ops from other doc. /// /// After `a.merge(b)` and `b.merge(a)`, `a` and `b` will have the same content if they are in attached mode. @@ -2705,7 +2838,24 @@ impl LoroDoc { } pub fn query_pos(&self, pos: &Cursor) -> Result { - self.query_pos_internal(pos, true) + match self.query_pos_internal(pos, true) { + Ok(ans) => Ok(ans), + Err(QueryPosError::Missing(err)) => Err(err), + // `CannotFindRelativePosition` is `Copy` and has no room for the + // decode error. `HistoryUnreadable` is the non-panicking mapping. + Err(QueryPosError::History(_)) => Err(CannotFindRelativePosition::HistoryUnreadable), + } + } + + /// [`query_pos`](Self::query_pos), preserving a decode error from unreadable history. + pub fn try_get_cursor_pos(&self, pos: &Cursor) -> LoroResult { + match self.query_pos_internal(pos, true) { + Ok(ans) => Ok(ans), + Err(QueryPosError::History(err)) => Err(err), + Err(QueryPosError::Missing(err)) => { + Err(LoroError::NotFoundError(err.to_string().into_boxed_str())) + } + } } /// Get position in a seq container @@ -2713,9 +2863,9 @@ impl LoroDoc { &self, pos: &Cursor, ret_event_index: bool, - ) -> Result { + ) -> Result { if !self.has_container(&pos.container) { - return Err(CannotFindRelativePosition::IdNotFound); + return Err(CannotFindRelativePosition::IdNotFound.into()); } // Cursors can only point into sequence containers @@ -2723,7 +2873,7 @@ impl LoroDoc { pos.container.container_type(), ContainerType::Text | ContainerType::List | ContainerType::MovableList ) { - return Err(CannotFindRelativePosition::IdNotFound); + return Err(CannotFindRelativePosition::IdNotFound.into()); } let mut state = self.state.lock(); @@ -2756,34 +2906,34 @@ impl LoroDoc { if oplog.arena.id_to_idx(&pos.container).is_none() { let mut s = self.state.lock(); if !s.does_container_exist(&pos.container) { - return Err(CannotFindRelativePosition::ContainerDeleted); + return Err(CannotFindRelativePosition::ContainerDeleted.into()); } s.ensure_container(&pos.container); drop(s); } let idx = oplog.arena.id_to_idx(&pos.container).unwrap(); // We know where the target id is when we trace back to the delete_op_id. - let Some(delete_op_id) = find_last_delete_op(&oplog, id, idx) else { + let Some(delete_op_id) = find_last_delete_op(&oplog, id, idx)? else { if oplog.shallow_since_vv().includes_id(id) { - return Err(CannotFindRelativePosition::HistoryCleared); + return Err(CannotFindRelativePosition::HistoryCleared.into()); } tracing::error!("Cannot find id {}", id); - return Err(CannotFindRelativePosition::IdNotFound); + return Err(CannotFindRelativePosition::IdNotFound.into()); }; // Should use persist mode so that it will force all the diff calculators to use the `checkout` mode let mut diff_calc = DiffCalculator::new(true); let before_frontiers: Frontiers = oplog.dag.find_deps_of_id(delete_op_id); - let before = &oplog.dag.frontiers_to_vv(&before_frontiers).unwrap(); + let before = frontiers_to_vv_or_not_found(&oplog, &before_frontiers)?; // TODO: PERF: it doesn't need to calc the effects here - diff_calc.calc_diff_internal( + diff_calc.try_calc_diff_internal( &oplog, - before, + &before, &before_frontiers, oplog.vv(), oplog.frontiers(), Some(&|target| idx == target), - ); + )?; // TODO: remove depth info let depth = self.arena.get_depth(idx); let (_, diff_calc) = &mut diff_calc.get_or_create_calc(idx, depth); @@ -3021,6 +3171,22 @@ impl LoroDoc { self.oplog().lock().dag.find_path(from, to) } + /// Finds operation spans between versions, reporting unreadable history. + pub fn try_find_id_spans_between( + &self, + from: &Frontiers, + to: &Frontiers, + ) -> LoroResult { + let oplog = self.oplog.lock(); + oplog.check_history_parsable()?; + for id in from.iter().chain(to.iter()) { + if !oplog.vv().includes_id(id) { + return Err(LoroError::FrontiersNotFound(id)); + } + } + oplog.dag.try_find_path(from, to) + } + /// Subscribe to the first commit from a peer. Operations performed on the `LoroDoc` within this callback /// will be merged into the current commit. /// @@ -3066,7 +3232,10 @@ fn materialize_touched_roots(state: &mut DocState, calc: &DiffCalculator, arena: } } -fn pending_root_containers_to_materialize(oplog: &OpLog, changes: &[Change]) -> Vec { +fn pending_root_containers_to_materialize( + oplog: &OpLog, + changes: &[Change], +) -> LoroResult> { let mut roots = FxHashSet::default(); for change in changes { if change.ctr_end() <= oplog.vv().get(&change.id.peer).copied().unwrap_or(0) { @@ -3076,7 +3245,7 @@ fn pending_root_containers_to_materialize(oplog: &OpLog, changes: &[Change]) -> if oplog.dag.is_before_shallow_root(&change.deps) || oplog .dag - .get_change_lamport_from_deps(&change.deps) + .try_get_change_lamport_from_deps(&change.deps)? .is_some() { continue; @@ -3101,7 +3270,25 @@ fn pending_root_containers_to_materialize(oplog: &OpLog, changes: &[Change]) -> } } - roots.into_iter().collect() + Ok(roots.into_iter().collect()) +} + +/// Version vector of `frontiers`. `None` from the dag walk is a missing id, not +/// an empty vector and not a panic. +fn frontiers_to_vv_or_not_found(oplog: &OpLog, frontiers: &Frontiers) -> LoroResult { + match oplog.dag.try_frontiers_to_vv(frontiers)? { + Some(vv) => Ok(vv), + None => { + for id in frontiers.iter() { + if oplog.dag.try_get(id)?.is_none() { + return Err(LoroError::FrontiersNotFound(id)); + } + } + Err(LoroError::NotFoundError( + format!("Cannot find the specified version {frontiers:?}").into_boxed_str(), + )) + } + } } /// Identify a causally closed update component that can be applied without replaying unrelated @@ -3181,7 +3368,10 @@ fn isolated_scalar_root_batch( } #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum ChangeTravelError { + #[error(transparent)] + HistoryUnreadable(#[from] LoroError), #[error("Target id not found {0:?}")] TargetIdNotFound(ID), #[error("The shallow history of the doc doesn't include the target version")] @@ -3219,43 +3409,49 @@ impl LoroDoc { } } - for id in ids { - let op_log = &self.oplog().lock(); - if !op_log.vv().includes_id(*id) { - return Err(ChangeTravelError::TargetIdNotFound(*id)); - } - if op_log.dag.shallow_since_vv().includes_id(*id) { - return Err(ChangeTravelError::TargetVersionNotIncluded); + let ans = (|| { + for id in ids { + let op_log = &self.oplog().lock(); + op_log.check_history_parsable()?; + if !op_log.vv().includes_id(*id) { + return Err(ChangeTravelError::TargetIdNotFound(*id)); + } + if op_log.dag.shallow_since_vv().includes_id(*id) { + return Err(ChangeTravelError::TargetVersionNotIncluded); + } } - } - let mut visited = FxHashSet::default(); - let mut pending: BinaryHeap = BinaryHeap::new(); - for id in ids { - pending.push(PendingNode(ChangeMeta::from_change( - &self.oplog().lock().get_change_at(*id).unwrap(), - ))); - } - while let Some(PendingNode(node)) = pending.pop() { - let deps = node.deps.clone(); - if f(node).is_break() { - break; + let mut visited = FxHashSet::default(); + let mut pending: BinaryHeap = BinaryHeap::new(); + for id in ids { + let change = self + .oplog() + .lock() + .try_get_change_at(*id)? + .ok_or(ChangeTravelError::TargetIdNotFound(*id))?; + pending.push(PendingNode(ChangeMeta::from_change(&change))); } - - for dep in deps.iter() { - let Some(dep_node) = self.oplog().lock().get_change_at(dep) else { - continue; - }; - if visited.contains(&dep_node.id) { - continue; + while let Some(PendingNode(node)) = pending.pop() { + let deps = node.deps.clone(); + if f(node).is_break() { + break; } - visited.insert(dep_node.id); - pending.push(PendingNode(ChangeMeta::from_change(&dep_node))); + for dep in deps.iter() { + let Some(dep_node) = self.oplog().lock().try_get_change_at(dep)? else { + continue; + }; + if visited.contains(&dep_node.id) { + continue; + } + + visited.insert(dep_node.id); + pending.push(PendingNode(ChangeMeta::from_change(&dep_node))); + } } - } - let ans = Ok(()); + Ok(()) + })(); self.renew_txn_if_auto_commit(options); ans } @@ -3288,6 +3484,18 @@ impl LoroDoc { }) } + /// Gets modified container IDs, reporting unreadable history. + pub fn try_get_changed_containers_in( + &self, + id: ID, + len: usize, + ) -> LoroResult> { + self.oplog.lock().check_history_parsable()?; + let containers = self.get_changed_containers_in(id, len); + self.oplog.lock().check_history_parsable()?; + Ok(containers) + } + pub fn delete_root_container(&self, cid: ContainerID) { if !cid.is_root() { return; @@ -3319,7 +3527,24 @@ impl LoroDoc { } } -fn find_last_delete_op(oplog: &OpLog, id: ID, idx: ContainerIdx) -> Option { +pub(crate) enum QueryPosError { + Missing(CannotFindRelativePosition), + History(LoroError), +} + +impl From for QueryPosError { + fn from(err: LoroError) -> Self { + QueryPosError::History(err) + } +} + +impl From for QueryPosError { + fn from(err: CannotFindRelativePosition) -> Self { + QueryPosError::Missing(err) + } +} + +fn find_last_delete_op(oplog: &OpLog, id: ID, idx: ContainerIdx) -> LoroResult> { // Any delete op that covers `id` must have observed it, so its peer's counter // at delete time was > id.counter. start_vv (the vv at `id`) is therefore a // valid lower bound: changes at or before start_vv[peer] predate `id` and can @@ -3330,10 +3555,10 @@ fn find_last_delete_op(oplog: &OpLog, id: ID, idx: ContainerIdx) -> Option { // ordering. op_lamport is the Lamport of the specific op within the change // (change.lamport + op offset), not just the change's starting Lamport, so // concurrent deletes with equal change Lamports are broken deterministically. - let start_vv = oplog - .dag - .frontiers_to_vv(&id.into()) - .unwrap_or_else(|| oplog.shallow_since_vv().to_vv()); + let start_vv = match oplog.dag.try_frontiers_to_vv(&id.into())? { + Some(vv) => vv, + None => oplog.shallow_since_vv().to_vv(), + }; // (op_lamport, peer) gives a deterministic total order for concurrent deletes. // A single peer cannot produce two ops with the same lamport, so peer suffices @@ -3360,7 +3585,10 @@ fn find_last_delete_op(oplog: &OpLog, id: ID, idx: ContainerIdx) -> Option { } } - best.map(|(_, op_id)| op_id) + // The legacy peer iterator records a bad block and skips it. A skipped + // block may hold the delete, so that is a decode error, not a missing id. + oplog.check_history_parsable()?; + Ok(best.map(|(_, op_id)| op_id)) } /// Cleanup for the critical section of [`LoroDoc::import_batch`]. @@ -3398,21 +3626,21 @@ fn recalc_in_checkout_mode_if_needed( old_vv: &VersionVector, old_frontiers: &Frontiers, diff: Vec, -) -> Vec { +) -> LoroResult> { if !state.needs_checkout_diff(&diff) { - return diff; + return Ok(diff); } - DiffCalculator::new(true) - .calc_diff_internal( + Ok(DiffCalculator::new(true) + .try_calc_diff_internal( oplog, old_vv, old_frontiers, oplog.vv(), oplog.dag.get_frontiers(), None, - ) - .0 + )? + .0) } impl BatchImportGuard<'_> { @@ -3697,7 +3925,9 @@ mod test { list.insert_with_txn(&mut txn, 1, "tail".into()).unwrap(); txn.commit().unwrap(); - let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false); + let json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv(), false) + .unwrap(); assert_eq!(json.changes.len(), 1); assert_eq!(json.changes[0].ops.len(), 4); (doc, json) @@ -3799,7 +4029,9 @@ mod test { #[test] fn failed_import_json_updates_rolls_back_complex_empty_doc() { let src = make_json_import_stress_doc(11); - let json = src.export_json_updates(&Default::default(), &src.oplog_vv(), false); + let json = src + .export_json_updates(&Default::default(), &src.oplog_vv(), false) + .unwrap(); let dst = LoroDoc::new(); let vv_before_import = dst.oplog_vv(); @@ -3836,7 +4068,9 @@ mod test { tree.get_meta(root).unwrap().insert("name", "root").unwrap(); let first_vv = src.oplog_vv(); - let first_json = src.export_json_updates(&Default::default(), &first_vv, false); + let first_json = src + .export_json_updates(&Default::default(), &first_vv, false) + .unwrap(); let mut text_pos = text.len_unicode(); for i in 0..64 { @@ -3855,7 +4089,9 @@ mod test { .insert("name", "child") .unwrap(); - let second_json = src.export_json_updates(&first_vv, &src.oplog_vv(), false); + let second_json = src + .export_json_updates(&first_vv, &src.oplog_vv(), false) + .unwrap(); let dst = LoroDoc::new(); dst.import_json_updates(first_json).unwrap(); @@ -3891,12 +4127,16 @@ mod test { let last_op_counter = good_json.changes[0].ops.last().unwrap().counter; let prefix_vv = VersionVector::from_iter([(peer, last_op_counter)]); - let prefix_json = src.export_json_updates(&Default::default(), &prefix_vv, false); + let prefix_json = src + .export_json_updates(&Default::default(), &prefix_vv, false) + .unwrap(); assert_eq!( prefix_json.changes[0].ops.len(), good_json.changes[0].ops.len() - 1 ); - let good_suffix_json = src.export_json_updates(&prefix_vv, &src.oplog_vv(), false); + let good_suffix_json = src + .export_json_updates(&prefix_vv, &src.oplog_vv(), false) + .unwrap(); assert_eq!(good_suffix_json.changes[0].ops.len(), 1); let mut bad_suffix_json = good_suffix_json.clone(); move_last_list_insert_far_out_of_bounds(&mut bad_suffix_json); diff --git a/crates/loro-internal/src/oplog.rs b/crates/loro-internal/src/oplog.rs index 260f3713d..5e58fd875 100644 --- a/crates/loro-internal/src/oplog.rs +++ b/crates/loro-internal/src/oplog.rs @@ -272,6 +272,17 @@ impl OpLog { Some(ans) } + pub fn try_get_change_with_lamport_lte( + &self, + peer: PeerID, + lamport: Lamport, + ) -> Result, LoroError> { + self.check_history_parsable()?; + let change = self.get_change_with_lamport_lte(peer, lamport); + self.check_history_parsable()?; + Ok(change) + } + pub fn get_timestamp_of_version(&self, f: &Frontiers) -> Timestamp { let mut timestamp = Timestamp::default(); for id in f.iter() { @@ -348,11 +359,21 @@ impl OpLog { } pub(crate) fn begin_import_rollback_with_arena(&mut self, arena: SharedArenaRollback) { + let old_vv = Arc::new(self.vv().clone()); + self.begin_import_rollback_with_version(arena, old_vv); + } + + /// Share the attached import's existing version snapshot with both journals. + pub(crate) fn begin_import_rollback_with_version( + &mut self, + arena: SharedArenaRollback, + old_vv: Arc, + ) { debug_assert!(self.import_rollback.is_none()); - self.dag.begin_import_rollback(); + self.dag.begin_import_rollback(old_vv.clone()); self.import_rollback = Some(ImportRollback { arena, - change_store: ChangeStoreRollback::new(self.vv().clone()), + change_store: ChangeStoreRollback::new(old_vv), pending: Default::default(), movable_list_refs: Vec::new(), }); @@ -394,14 +415,14 @@ impl OpLog { &self, changes: &[Change], detached: bool, - ) -> ImportChangesPreflight { + ) -> Result { let mut ans = ImportChangesPreflight::default(); for change in changes { if change.ctr_end() <= self.vv().get(&change.id.peer).copied().unwrap_or(0) { continue; } - if self.dag.import_deps_before_shallow_root(&change.deps) { + if self.dag.try_import_deps_before_shallow_root(&change.deps)? { ans.has_deps_before_shallow_root = true; continue; } @@ -418,7 +439,7 @@ impl OpLog { if self .dag - .get_change_lamport_from_deps(&change.deps) + .try_get_change_lamport_from_deps(&change.deps)? .is_some() { ans.applies_to_dag = true; @@ -449,7 +470,7 @@ impl OpLog { ans.needs_state_apply_rollback = true; } - ans + Ok(ans) } /// Undo the open import rollback scope, including the arena. @@ -696,6 +717,13 @@ impl OpLog { self.change_store.get_change(id) } + pub fn try_get_change_at(&self, id: ID) -> Result, LoroError> { + self.check_history_parsable()?; + let change = self.get_change_at(id); + self.check_history_parsable()?; + Ok(change) + } + pub(crate) fn set_uncommitted_change(&mut self, change: Change) { self.uncommitted_change = Some(change); } @@ -730,6 +758,14 @@ impl OpLog { Some(convert_change_to_remote(&self.arena, &change)) } + pub fn try_get_remote_change_at( + &self, + id: ID, + ) -> Result>>, LoroError> { + let change = self.try_get_change_at(id)?; + Ok(change.map(|change| convert_change_to_remote(&self.arena, &change))) + } + pub(crate) fn import_unknown_lamport_pending_changes( &mut self, remote_changes: Vec, @@ -773,12 +809,18 @@ impl OpLog { self.change_store.export_blocks_in_range(spans, w) } - pub(crate) fn fork_changes_up_to(&self, frontiers: &Frontiers) -> Option { - let vv = self.dag.frontiers_to_vv(frontiers)?; - Some( - self.change_store - .fork_changes_up_to(self.dag.shallow_since_vv(), frontiers, &vv), - ) + pub(crate) fn fork_changes_up_to( + &self, + frontiers: &Frontiers, + ) -> Result, LoroError> { + let Some(vv) = self.dag.try_frontiers_to_vv(frontiers)? else { + return Ok(None); + }; + Ok(Some(self.change_store.fork_changes_up_to( + self.dag.shallow_since_vv(), + frontiers, + &vv, + )?)) } #[inline(always)] @@ -790,16 +832,16 @@ impl OpLog { /// /// Only `op.container` is needed, so this scans the borrowed changes /// directly instead of materializing a cloned RichOp per op. - pub(crate) fn containers_in_spans( + pub(crate) fn try_containers_in_spans( &self, spans: impl Iterator, - ) -> FxHashSet { + ) -> Result, LoroError> { let mut containers = FxHashSet::default(); // Consecutive ops overwhelmingly share a container; skip the hash // probe when it hasn't changed. let mut last_container = None; for span in spans { - for change in self.change_store.iter_changes(span) { + for change in self.change_store.try_iter_changes(span)? { let start_counter = span.counter.min().max(change.id.counter); let end_counter = span.counter.norm_end(); let start = change @@ -819,7 +861,7 @@ impl OpLog { } } - containers + Ok(containers) } /// For `to ⊇ from`: the old-parent frontiers of every entry change of the @@ -844,11 +886,11 @@ impl OpLog { from: &VersionVector, from_frontiers: &Frontiers, to: &VersionVector, - ) -> Option> { + ) -> Result>, LoroError> { let mut uncovered = Vec::new(); for (peer, span) in from.diff(to).forward.iter() { let id_span = IdSpan::new(*peer, span.start, span.end); - for change in self.change_store.iter_changes(id_span) { + for change in self.change_store.try_iter_changes(id_span)? { let start = change.id.counter.max(span.start); let parents = if start > change.id.counter { // The change straddles `from`; the new suffix only has @@ -877,14 +919,16 @@ impl OpLog { continue; } - let parents_vv = self.dag.frontiers_to_vv(&parents)?; + let Some(parents_vv) = self.dag.try_frontiers_to_vv(&parents)? else { + return Ok(None); + }; if !parents_vv.includes_vv(from) { uncovered.push(parents); } } } - Some(uncovered) + Ok(Some(uncovered)) } /// Decide whether an import whose new region `to − from` is concurrent @@ -912,11 +956,13 @@ impl OpLog { from: &VersionVector, to: &VersionVector, entry_parents: &[Frontiers], - ) -> Option> { + ) -> Result>, LoroError> { // ⋂ Events(parents) as a version vector: per-peer minimum. let mut causal_past = from.clone(); for parents in entry_parents { - let parents_vv = self.dag.frontiers_to_vv(parents)?; + let Some(parents_vv) = self.dag.try_frontiers_to_vv(parents)? else { + return Ok(None); + }; causal_past.retain(|peer, end| { let bound = parents_vv.get(peer).copied().unwrap_or(0); *end = (*end).min(bound); @@ -930,25 +976,25 @@ impl OpLog { // that trimmed history we cannot see which containers it touched, so // the decision has to stay with the DAG. if !causal_past.includes_vv(&self.dag.shallow_since_vv().to_vv()) { - return None; + return Ok(None); } let concurrent_old = causal_past.diff(from).forward; - let old_containers = self.containers_in_spans( + let old_containers = self.try_containers_in_spans( concurrent_old .iter() .map(|(peer, span)| IdSpan::new(*peer, span.start, span.end)), - ); + )?; if old_containers.is_empty() { - return Some(old_containers); + return Ok(Some(old_containers)); } let new_region = from.diff(to).forward; - let new_containers = self.containers_in_spans( + let new_containers = self.try_containers_in_spans( new_region .iter() .map(|(peer, span)| IdSpan::new(*peer, span.start, span.end)), - ); + )?; let harmless = old_containers .iter() .filter(|idx| new_containers.contains(idx)) @@ -958,7 +1004,7 @@ impl OpLog { ContainerType::Counter => true, _ => false, }); - harmless.then_some(old_containers) + Ok(harmless.then_some(old_containers)) } /// The latest critical version below `from ∩ to`: the greatest causally @@ -986,31 +1032,42 @@ impl OpLog { /// that moves its peer's cut into it. The scan is thus bounded by the /// replay it enables, which walks the same region and computes the same /// contexts, so it needs no budget of its own. + #[cfg(test)] pub(crate) fn latest_critical_version_below_meet( &self, from: &VersionVector, to: &VersionVector, merged: &VersionVector, ) -> CriticalVersionSearch { + self.try_latest_critical_version_below_meet(from, to, merged) + .unwrap() + } + + pub(crate) fn try_latest_critical_version_below_meet( + &self, + from: &VersionVector, + to: &VersionVector, + merged: &VersionVector, + ) -> Result { use CriticalVersionSearch::*; // An intersection of two causally closed sets is causally closed, so // `min(from, to)` is a version, and it is the meet of the two. let mut v = from.intersection(to); if v.is_empty() { - return NoneBelowMeet; + return Ok(NoneBelowMeet); } let mut pending: Vec = v.diff_iter(merged).1.collect(); while let Some(span) = pending.pop() { - for change in self.change_store.iter_changes(span) { + for change in self.change_store.try_iter_changes(span)? { // The context of the change's first op above `v`, as // `iter_from_replay_base_causally` computes it for the // replay: the recorded deps plus the same-peer prefix. - let Some(mut ctx) = self.dag.frontiers_to_vv(&change.deps) else { + let Some(mut ctx) = self.dag.try_frontiers_to_vv(&change.deps)? else { // The deps reach below trimmed history; nothing can be // proved from here. - return Unknown; + return Ok(Unknown); }; let cut = v.get(&change.id.peer).copied().unwrap_or(0); let first_above = change.id.counter.max(cut); @@ -1020,14 +1077,14 @@ impl OpLog { let above = v.clone(); v.intersect_with(&ctx); if v.is_empty() { - return NoneBelowMeet; + return Ok(NoneBelowMeet); } pending.extend(v.diff_iter(&above).1); } } } - Found(v) + Ok(Found(v)) } /// Iterates causally over all changes between the replay base (the meet @@ -1068,6 +1125,35 @@ impl OpLog { ), > + '_, ) { + let (base, iter) = self + .try_iter_from_replay_base_causally(from, from_frontiers, to, to_frontiers) + .unwrap(); + (base, iter.map(Result::unwrap)) + } + + #[allow(clippy::type_complexity)] + pub(crate) fn try_iter_from_replay_base_causally( + &self, + from: &VersionVector, + from_frontiers: &Frontiers, + to: &VersionVector, + to_frontiers: &Frontiers, + ) -> Result< + ( + ReplayBase, + impl Iterator< + Item = Result< + ( + BlockChangeRef, + (Counter, Counter), + Rc>, + ), + LoroError, + >, + > + '_, + ), + LoroError, + > { let mut merged_vv = from.clone(); merged_vv.merge(to); loro_common::debug!("to_frontiers={:?} vv={:?}", &to_frontiers, to); @@ -1079,10 +1165,10 @@ impl OpLog { // to a critical version; with container knowledge we can often // prove the concurrency harmless and replay from `from` // without touching the DAG at all. - if let Some(entry_parents) = self.uncovered_entry_parents(from, from_frontiers, to) { + if let Some(entry_parents) = self.uncovered_entry_parents(from, from_frontiers, to)? { if !entry_parents.is_empty() { if let Some(containers) = - self.register_only_concurrency(from, to, &entry_parents) + self.register_only_concurrency(from, to, &entry_parents)? { concurrent_containers = Some(containers); register_only_base = Some(from_frontiers.clone()); @@ -1094,7 +1180,8 @@ impl OpLog { let (meet, mut diff_mode) = if let Some(base) = register_only_base { (MeetAsBase::Valid(base), DiffMode::ImportGreaterUpdates) } else { - self.dag.find_meet_and_mode(from_frontiers, to_frontiers) + self.dag + .try_find_meet_and_mode(from_frontiers, to_frontiers)? }; if diff_mode == DiffMode::Checkout && to > from { diff_mode = DiffMode::Import; @@ -1114,9 +1201,9 @@ impl OpLog { #[cfg(test)] CRITICAL_BASE_FALLBACK_COUNT.with(|c| c.set(c.get() + 1)); self.dag - .latest_single_head_critical_version(from_frontiers, to_frontiers) + .try_latest_single_head_critical_version(from_frontiers, to_frontiers) }; - match self.latest_critical_version_below_meet(from, to, &merged_vv) { + match self.try_latest_critical_version_below_meet(from, to, &merged_vv)? { CriticalVersionSearch::Found(v) => { // Only replay from the version whose criticality was // proved. A shallow doc cannot replay from below its @@ -1127,17 +1214,18 @@ impl OpLog { // against) — a version that fails to round-trip // through frontiers is not one. Both are left to the // descent, as before. - let f = self.dag.vv_to_frontiers(&v); + let f = self.dag.try_vv_to_frontiers(&v)?; let seed = self .dag - .frontiers_to_vv(self.dag.shallow_since_frontiers()) + .try_frontiers_to_vv(self.dag.shallow_since_frontiers())? .unwrap(); - if v.includes_vv(&seed) && self.dag.frontiers_to_vv(&f).as_ref() == Some(&v) + if v.includes_vv(&seed) + && self.dag.try_frontiers_to_vv(&f)?.as_ref() == Some(&v) { replay_base_is_critical = true; f } else { - descend() + descend()? } } CriticalVersionSearch::NoneBelowMeet => { @@ -1148,12 +1236,15 @@ impl OpLog { // walk. Frontiers::default() } - CriticalVersionSearch::Unknown => descend(), + CriticalVersionSearch::Unknown => descend()?, } } }; - let mut replay_base_vv = self.dag.frontiers_to_vv(&replay_base_frontiers).unwrap(); + let mut replay_base_vv = self + .dag + .try_frontiers_to_vv(&replay_base_frontiers)? + .unwrap(); replay_base_is_critical |= replay_base_vv.is_empty(); if !replay_base_vv.includes_vv(&shallow_since_vv) { // The replay base cannot point before shallow history because those @@ -1161,18 +1252,18 @@ impl OpLog { replay_base_frontiers = self.dag.shallow_since_frontiers().clone(); replay_base_vv = self .dag - .frontiers_to_vv(&replay_base_frontiers) + .try_frontiers_to_vv(&replay_base_frontiers)? .unwrap_or(shallow_since_vv); replay_base_is_critical = false; } // go from the replay base to merged_vv let diff = replay_base_vv.diff(&merged_vv).forward; - let mut iter = self.dag.iter_causal(replay_base_frontiers, diff); + let mut iter = crate::dag::DagCausalIter::try_new(&self.dag, replay_base_frontiers, diff)?; let mut node = iter.next(); let mut cur_cnt = 0; let vv = Rc::new(RefCell::new(VersionVector::default())); - ( + Ok(( ReplayBase { vv: replay_base_vv.clone(), diff_mode, @@ -1180,37 +1271,52 @@ impl OpLog { concurrent_containers, }, std::iter::from_fn(move || { - if let Some(inner) = &node { - let mut inner_vv = vv.borrow_mut(); - // FIXME: PERF: it looks slow for large vv, like 10000+ entries - inner_vv.clear(); - self.dag.ensure_vv_for(&inner.data); - inner_vv.extend_to_include_vv(inner.data.vv.get().unwrap().iter()); - let peer = inner.data.peer; - let cnt = inner - .data - .cnt - .max(cur_cnt) - .max(replay_base_vv.get(&peer).copied().unwrap_or(0)); - let dag_node_end = (inner.data.cnt + inner.data.len as Counter) - .min(merged_vv.get(&peer).copied().unwrap_or(0)); - let change = self.change_store.get_change(ID::new(peer, cnt)).unwrap(); - - if change.ctr_end() < dag_node_end { - cur_cnt = change.ctr_end(); - } else { - node = iter.next(); - cur_cnt = 0; - } + let result = (|| -> Result<_, LoroError> { + if let Some(inner) = &node { + let mut inner_vv = vv.borrow_mut(); + // FIXME: PERF: it looks slow for large vv, like 10000+ entries + inner_vv.clear(); + self.dag.try_ensure_vv_for(&inner.data)?; + inner_vv.extend_to_include_vv(inner.data.vv.get().unwrap().iter()); + let peer = inner.data.peer; + let cnt = inner + .data + .cnt + .max(cur_cnt) + .max(replay_base_vv.get(&peer).copied().unwrap_or(0)); + let dag_node_end = (inner.data.cnt + inner.data.len as Counter) + .min(merged_vv.get(&peer).copied().unwrap_or(0)); + let change = match self.change_store.get_change(ID::new(peer, cnt)) { + Some(change) => change, + None => { + self.check_history_parsable()?; + panic!("DAG node must have a change"); + } + }; + + if change.ctr_end() < dag_node_end { + cur_cnt = change.ctr_end(); + } else { + node = iter.next(); + cur_cnt = 0; + } - inner_vv.extend_to_include_end_id(change.id); + inner_vv.extend_to_include_end_id(change.id); - Some((change, (cnt, dag_node_end), vv.clone())) - } else { - None + Ok(Some((change, (cnt, dag_node_end), vv.clone()))) + } else { + Ok(None) + } + })(); + match result { + Ok(item) => item.map(Ok), + Err(err) => { + node = None; + Some(Err(err)) + } } }), - ) + )) } pub fn len_changes(&self) -> usize { @@ -1305,7 +1411,10 @@ impl OpLog { let elem = *elems .entry(r.elem_id) .or_insert_with(|| self.resolve_movable_list_elem(r.elem_id)); - if !self.is_visible_movable_list_elem(&r, elem) { + if elem.is_none() { + self.check_history_parsable()?; + } + if !self.is_visible_movable_list_elem(&r, elem)? { return Err(LoroError::DecodeError( format!( "Movable list op {} targets element {}, which is not in the list's \ @@ -1347,31 +1456,33 @@ impl OpLog { &self, r: &MovableListElemRef, elem: Option, - ) -> bool { + ) -> Result { if r.elem_id.lamport >= r.op_lamport { - return false; + return Ok(false); } let Some(elem) = elem else { // History before a shallow root is trimmed. An op after the root can // only see pre-root elements that are still alive at the root. - return !self.dag.shallow_since_vv().is_empty() + return Ok(!self.dag.shallow_since_vv().is_empty() && self.with_history_cache(|h| { h.shallow_root_has_movable_list_elem(r.container, r.elem_id) - }); + })); }; if !elem.is_insert || elem.container != r.container { - return false; + return Ok(false); } // Earlier ops of the op's own DAG node are by the same peer, so the node's // start version covers every other peer's part of its causal history. let target = elem.target; - (target.peer == r.op_id.peer && target.counter < r.op_id.counter) - || self - .dag - .get(r.op_id) - .is_some_and(|node| self.dag.ensure_vv_for(&node).includes_id(target)) + if target.peer == r.op_id.peer && target.counter < r.op_id.counter { + return Ok(true); + } + let Some(node) = self.dag.try_get(r.op_id)? else { + return Ok(false); + }; + Ok(self.dag.try_ensure_vv_for(&node)?.includes_id(target)) } #[allow(unused)] diff --git a/crates/loro-internal/src/oplog/change_store.rs b/crates/loro-internal/src/oplog/change_store.rs index 1644097cf..cbf6ffcf5 100644 --- a/crates/loro-internal/src/oplog/change_store.rs +++ b/crates/loro-internal/src/oplog/change_store.rs @@ -15,7 +15,6 @@ use crate::{ }; use block_encode::decode_block_range; use bytes::Bytes; -use itertools::Itertools; use loro_common::{ ContainerID, Counter, HasCounterSpan, HasId, HasIdSpan, HasLamportSpan, IdLp, IdSpan, Lamport, LoroError, LoroResult, PeerID, ID, @@ -163,11 +162,13 @@ struct ChangeStoreInner { /// Set by [`ChangeStore::retire`]: the op log replaced this store, so loading from it /// must not register anything in the arena any more. retired: bool, + /// Conservative until a full history walk: headers do not validate block bodies. + may_have_unparsed_bodies: bool, } #[derive(Debug)] pub(crate) struct ChangeStoreRollback { - old_vv: VersionVector, + old_vv: Arc, /// Pre-scope blocks that an import appended to, keyed by block id. `None` means the /// block was flushed, so its KV copy is the pre-scope version. blocks_before_mutation: BTreeMap>, @@ -192,7 +193,7 @@ struct BlockShape { } impl ChangeStoreRollback { - pub(crate) fn new(old_vv: VersionVector) -> Self { + pub(crate) fn new(old_vv: Arc) -> Self { Self { old_vv, blocks_before_mutation: BTreeMap::new(), @@ -270,6 +271,7 @@ impl ChangeStore { start_frontiers: Frontiers::default(), mem_parsed_kv: BTreeMap::new(), retired: false, + may_have_unparsed_bodies: false, })), arena: a.clone(), external_vv: Arc::new(Mutex::new(VersionVector::new())), @@ -370,7 +372,10 @@ impl ChangeStore { for span in layer { // PERF: this can be optimized by reusing the current encoded blocks // In the current method, it needs to parse and re-encode the blocks - for c in self.iter_changes(span) { + let Ok(changes) = self.try_iter_changes(span) else { + return; + }; + for c in changes { let start = ((span.counter.start - c.id.counter).max(0) as usize).min(c.atom_len()); let end = ((span.counter.end - c.id.counter).max(0) as usize).min(c.atom_len()); @@ -673,14 +678,17 @@ impl ChangeStore { } fn rollback_arena_in(&self, inner: &mut ChangeStoreInner, arena: SharedArenaRollback) { + let mut dropped_body = false; for block in inner.mem_parsed_kv.values_mut() { if let ChangesBlockContent::Both(_, bytes) = &block.content { if !arena.keeps(block.parsed_extent) { let bytes = bytes.clone(); Arc::make_mut(block).content = ChangesBlockContent::Bytes(bytes); + dropped_body = true; } } } + inner.may_have_unparsed_bodies |= dropped_body; self.arena.rollback(arena); } @@ -705,6 +713,12 @@ impl ChangeStore { Some(block.content.iter_dag_nodes()) } + /// Whether diff calculation may still encounter an unvalidated snapshot body. + /// Loading all DAG headers cannot clear this; only a successful full body walk can. + pub(crate) fn may_have_unparsed_bodies(&self) -> bool { + self.inner.lock().may_have_unparsed_bodies + } + pub fn visit_all_changes(&self, f: &mut dyn FnMut(&Change)) { self.ensure_block_loaded_in_range(Bound::Unbounded, Bound::Unbounded); let mut inner = self.inner.lock(); @@ -718,6 +732,9 @@ impl ChangeStore { f(c); } } + if self.corrupt_block_error().is_ok() { + inner.may_have_unparsed_bodies = false; + } } fn build_root_history_names(&self) -> RootHistoryNamesState { @@ -840,83 +857,86 @@ impl ChangeStore { } pub(crate) fn iter_blocks(&self, id_span: IdSpan) -> Vec<(Arc, usize, usize)> { + self.iter_blocks_inner(id_span, false).unwrap() + } + + fn try_iter_blocks( + &self, + id_span: IdSpan, + ) -> LoroResult, usize, usize)>> { + self.iter_blocks_inner(id_span, true) + } + + fn iter_blocks_inner( + &self, + id_span: IdSpan, + fail_on_corrupt: bool, + ) -> LoroResult, usize, usize)>> { if id_span.counter.start == id_span.counter.end { - return vec![]; + return Ok(vec![]); } - assert!(id_span.counter.start < id_span.counter.end); - self.ensure_block_loaded_in_range( - Bound::Included(id_span.id_start()), - Bound::Excluded(id_span.id_end()), - ); + if fail_on_corrupt { + self.try_ensure_block_loaded_in_range( + Bound::Included(id_span.id_start()), + Bound::Excluded(id_span.id_end()), + )?; + } else { + self.ensure_block_loaded_in_range( + Bound::Included(id_span.id_start()), + Bound::Excluded(id_span.id_end()), + ); + } let mut inner = self.inner.lock(); - let next_back = inner.mem_parsed_kv.range(..=id_span.id_start()).next_back(); - match next_back { - None => { - return vec![]; - } - Some(next_back) => { - if next_back.0.peer != id_span.peer { - return vec![]; + let start_counter = inner + .mem_parsed_kv + .range(..=id_span.id_start()) + .next_back() + .filter(|(id, _)| id.peer == id_span.peer) + .map_or(id_span.counter.start, |(id, _)| id.counter); + let mut blocks = Vec::new(); + for (block_id, block) in inner.mem_parsed_kv.range_mut( + ID::new(id_span.peer, start_counter)..ID::new(id_span.peer, id_span.counter.end), + ) { + if block.counter_range.1 < id_span.counter.start { + continue; + } + if let Err(err) = block.ensure_changes(&self.arena) { + warn!(?block_id, ?err, "failed to parse change block"); + self.parse_failures.record(*block_id, &err); + if fail_on_corrupt { + return Err(self.corrupt_block_error().unwrap_err()); } + continue; + } + let changes = block.content.try_changes().unwrap(); + let (start, end) = if id_span.counter.start <= block.counter_range.0 + && id_span.counter.end >= block.counter_range.1 + { + (0, changes.len()) + } else { + let start = block + .get_change_index_by_counter(id_span.counter.start) + .unwrap_or_else(|x| x); + let end = match block.get_change_index_by_counter(id_span.counter.end - 1) { + Ok(end) => end + 1, + Err(0) => continue, + Err(end) => end, + }; + (start, end) + }; + if start != end { + blocks.push((block.clone(), start, end)); } } - let start_counter = next_back.map(|(id, _)| id.counter).unwrap_or(0); - let ans = inner - .mem_parsed_kv - .range_mut( - ID::new(id_span.peer, start_counter)..ID::new(id_span.peer, id_span.counter.end), - ) - .filter_map(|(_id, block)| { - if block.counter_range.1 < id_span.counter.start { - return None; - } - - if let Err(err) = block.ensure_changes(&self.arena) { - warn!(block_id = ?_id, ?err, "failed to parse change block"); - self.parse_failures.record(*_id, &err); - return None; - } - let changes = block.content.try_changes().unwrap(); - let start; - let end; - if id_span.counter.start <= block.counter_range.0 - && id_span.counter.end >= block.counter_range.1 - { - start = 0; - end = changes.len(); - } else { - start = block - .get_change_index_by_counter(id_span.counter.start) - .unwrap_or_else(|x| x); - - match block.get_change_index_by_counter(id_span.counter.end - 1) { - Ok(e) => { - end = e + 1; - } - Err(0) => return None, - Err(e) => { - end = e; - } - } - } - if start == end { - return None; - } - - Some((block.clone(), start, end)) - }) - // TODO: PERF avoid alloc - .collect_vec(); - - ans + Ok(blocks) } pub fn iter_changes(&self, id_span: IdSpan) -> impl Iterator + '_ { let v = self.iter_blocks(id_span); #[cfg(debug_assertions)] { - if !v.is_empty() { + if !v.is_empty() && self.corrupt_block_error().is_ok() { assert_eq!(v[0].0.peer, id_span.peer); assert_eq!(v.last().unwrap().0.peer, id_span.peer); { @@ -943,6 +963,21 @@ impl ChangeStore { }) } + /// Parse the range before handing any changes to a consumer: a missing middle + /// block must not become a counter gap in an export's scratch store. + pub(crate) fn try_iter_changes( + &self, + id_span: IdSpan, + ) -> LoroResult> { + let blocks = self.try_iter_blocks(id_span)?; + Ok(blocks.into_iter().flat_map(|(block, start, end)| { + (start..end).map(move |change_index| BlockChangeRef { + change_index, + block: block.clone(), + }) + })) + } + #[allow(dead_code)] pub(crate) fn get_blocks_in_range(&self, id_span: IdSpan) -> VecDeque> { let mut inner = self.inner.lock(); @@ -1031,6 +1066,7 @@ impl ChangeStore { start_frontiers: inner.start_frontiers.clone(), mem_parsed_kv: BTreeMap::new(), retired: false, + may_have_unparsed_bodies: !vv.is_empty(), })), arena, external_vv: Arc::new(Mutex::new(self.external_vv.lock().clone())), @@ -1069,7 +1105,10 @@ impl ChangeStore { // PERF: this can be optimized by reusing the current encoded blocks // In the current method, it needs to parse and re-encode the blocks - for c in self.iter_changes(span) { + let Ok(changes) = self.try_iter_changes(span) else { + return; + }; + for c in changes { let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) as usize) .min(c.atom_len()); @@ -1092,7 +1131,7 @@ impl ChangeStore { start_vv: &ImVersionVector, frontiers: &Frontiers, vv: &VersionVector, - ) -> Bytes { + ) -> LoroResult { let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); for mut span in vv.sub_iter_im(start_vv) { let counter_lower_bound = start_vv.get(&span.peer).copied().unwrap_or(0); @@ -1104,7 +1143,7 @@ impl ChangeStore { // PERF: this can be optimized by reusing the current encoded blocks // In the current method, it needs to parse and re-encode the blocks - for c in self.iter_changes(span) { + for c in self.try_iter_changes(span)? { let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) as usize) .min(c.atom_len()); @@ -1118,7 +1157,7 @@ impl ChangeStore { } } - new_store.encode_all(vv, frontiers) + Ok(new_store.encode_all(vv, frontiers)) } } @@ -1181,6 +1220,7 @@ mod mut_external_kv { } } + self.inner.lock().may_have_unparsed_bodies = !vv.is_empty(); *self.external_vv.lock() = vv.clone(); let frontiers_bytes = self .external_kv @@ -1825,6 +1865,23 @@ mod mut_inner_kv { /// This is fast because we don't actually parse the content. // TODO: PERF: This method feels slow. pub(super) fn ensure_block_loaded_in_range(&self, start: Bound, end: Bound) { + let _ = self.load_blocks_in_range(start, end, false); + } + + pub(super) fn try_ensure_block_loaded_in_range( + &self, + start: Bound, + end: Bound, + ) -> LoroResult<()> { + self.load_blocks_in_range(start, end, true) + } + + fn load_blocks_in_range( + &self, + start: Bound, + end: Bound, + stop_on_error: bool, + ) -> LoroResult<()> { let mut whether_need_scan_backward = match start { Bound::Included(id) => Some(id), Bound::Excluded(id) => Some(id.inc(1)), @@ -1859,6 +1916,9 @@ mod mut_inner_kv { Err(err) => { warn!(?id, ?err, "failed to decode external change block"); self.parse_failures.record(id, &err); + if stop_on_error { + return Err(self.corrupt_block_error().unwrap_err()); + } continue; } }; @@ -1867,24 +1927,33 @@ mod mut_inner_kv { } if let Some(start_id) = whether_need_scan_backward { - self.ensure_id_lte(start_id); + if stop_on_error { + self.try_ensure_id_lte(start_id)?; + } else { + self.ensure_id_lte(start_id); + } } + Ok(()) } pub(super) fn ensure_id_lte(&self, id: ID) { + let _ = self.try_ensure_id_lte(id); + } + + pub(super) fn try_ensure_id_lte(&self, id: ID) -> LoroResult<()> { let kv = self.external_kv.lock(); let mut inner = self.inner.lock(); let Some((next_back_id, next_back_bytes)) = kv .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) .rfind(|(id, _)| id.len() == 12) else { - return; + return Ok(()); }; let next_back_id = ID::from_bytes(&next_back_id); if next_back_id.peer == id.peer { if inner.mem_parsed_kv.contains_key(&next_back_id) { - return; + return Ok(()); } let block = match ChangesBlock::from_bytes(next_back_bytes) { @@ -1896,11 +1965,12 @@ mod mut_inner_kv { "failed to decode external change block" ); self.parse_failures.record(next_back_id, &err); - return; + return Err(self.corrupt_block_error().unwrap_err()); } }; inner.mem_parsed_kv.insert(next_back_id, Arc::new(block)); } + Ok(()) } } } @@ -2688,6 +2758,61 @@ mod test { store.inner.lock().mem_parsed_kv.clear(); } + #[test] + fn legacy_block_iteration_skips_only_the_broken_block() { + for position in 0..3 { + let (store, end, _) = kv_only_store_and_next_change(); + let ids: Vec<_> = store + .external_kv + .lock() + .scan(Bound::Unbounded, Bound::Unbounded) + .filter(|(key, _)| key.len() == 12) + .map(|(key, _)| ID::from_bytes(&key)) + .collect(); + assert!(ids.len() >= 3); + let broken = ids[match position { + 0 => 0, + 1 => ids.len() / 2, + _ => ids.len() - 1, + }]; + truncate_block(&store, broken); + let span = IdSpan::new(1, 0, end); + let blocks = store.iter_blocks(span); + assert_eq!(blocks.len(), ids.len() - 1, "position={position}"); + assert!(blocks + .iter() + .all(|(block, _, _)| block.counter_range.0 != broken.counter)); + assert!(store + .corrupt_block_error() + .unwrap_err() + .to_string() + .contains("cannot parse change block")); + // The legacy iterator also yields the healthy portions, without bounds asserts. + assert!(store.iter_changes(span).count() > 0); + assert!(store + .try_iter_blocks(span) + .unwrap_err() + .to_string() + .contains("cannot parse change block")); + } + } + + #[test] + fn body_validation_stays_lazy_after_header_reads_and_can_be_reset_by_rollback() { + let (store, end, _) = kv_only_store_and_next_change(); + assert!(store.may_have_unparsed_bodies()); + assert!(store.get_dag_nodes_that_contains(ID::new(1, 0)).is_some()); + assert!(store.may_have_unparsed_bodies()); + let arena = store.arena.checkpoint_for_rollback(); + store.visit_all_changes(&mut |_| {}); + assert!(!store.may_have_unparsed_bodies()); + store.rollback_arena(arena); + assert!(store.may_have_unparsed_bodies()); + assert!(store.get_change(ID::new(1, end - 1)).is_some()); + store.visit_all_changes(&mut |_| {}); + assert!(!store.may_have_unparsed_bodies()); + } + #[test] fn the_creator_resolver_reports_an_unparsable_block_instead_of_panicking() { let (store, _, _) = kv_only_store_and_next_change(); @@ -2799,6 +2924,69 @@ mod test { assert_eq!(doc.get_deep_value(), value); } + #[test] + fn the_first_dag_reader_reports_unparsable_tree_history_without_changing_state() { + let (snapshot, _, peer) = snapshot_with_a_container_only_the_history_knows(); + for api in ["checkout", "diff", "revert", "frontiers", "travel", "cmp"] { + let doc = LoroDoc::new(); + doc.import(&snapshot).unwrap(); + truncate_block(&doc.oplog().lock().change_store, ID::new(peer, 0)); + assert!(doc.oplog().lock().check_history_parsable().is_ok()); + let value = doc.get_deep_value(); + let state_frontiers = doc.state_frontiers(); + let mid = Frontiers::from(ID::new(peer, 10)); + let result = match api { + "checkout" => doc.checkout(&mid).map_err(|err| err.to_string()), + "diff" => doc + .diff(&mid, &state_frontiers) + .map(|_| ()) + .map_err(|err| err.to_string()), + "revert" => doc.revert_to(&mid).map_err(|err| err.to_string()), + "frontiers" => doc + .try_frontiers_to_vv(&mid) + .map(|_| ()) + .map_err(|err| err.to_string()), + "travel" => doc + .travel_change_ancestors(&[ID::new(peer, 10)], &mut |_| { + std::ops::ControlFlow::Continue(()) + }) + .map_err(|err| err.to_string()), + "cmp" => doc + .cmp_frontiers(&mid, &state_frontiers) + .map(|_| ()) + .map_err(|err| err.to_string()), + _ => unreachable!(), + }; + assert!( + result.unwrap_err().contains("cannot parse change block"), + "{api}" + ); + assert_eq!(doc.get_deep_value(), value, "{api}"); + assert_eq!(doc.state_frontiers(), state_frontiers, "{api}"); + assert!(matches!( + doc.oplog().lock().check_history_parsable(), + Err(LoroError::DecodeError(_)) + )); + } + } + + #[test] + fn a_detached_fork_records_unparsable_history_before_its_infallible_boundary_panics() { + let (snapshot, _, peer) = snapshot_with_a_container_only_the_history_knows(); + let doc = LoroDoc::new(); + doc.import(&snapshot).unwrap(); + truncate_block(&doc.oplog().lock().change_store, ID::new(peer, 0)); + let value = doc.get_deep_value(); + doc.detach(); + assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| doc.fork())).is_err()); + assert_eq!(doc.get_deep_value(), value); + assert!(doc + .checkout(&doc.oplog_frontiers()) + .unwrap_err() + .to_string() + .contains("cannot parse change block")); + } + #[test] fn a_recorded_parse_failure_does_not_panic_where_no_error_can_be_returned() { // `undo`, `checkout_to_latest` and a detached `fork` run the same checkout and export @@ -2863,7 +3051,7 @@ mod test { let (store, end, next) = kv_only_store_and_next_change(); let mut old_vv = VersionVector::new(); old_vv.insert(1, end); - let mut rollback = ChangeStoreRollback::new(old_vv); + let mut rollback = ChangeStoreRollback::new(Arc::new(old_vv)); let arena = store.arena.checkpoint_for_rollback(); store.insert_change_with_rollback(next.clone(), true, false, &mut rollback); assert!(store.get_change(ID::new(1, 0)).is_some()); @@ -2913,7 +3101,7 @@ mod test { for c in first { rolled_back.insert_change(c.clone(), true, false); } - let mut rollback = ChangeStoreRollback::new(vv_of(first)); + let mut rollback = ChangeStoreRollback::new(Arc::new(vv_of(first))); let arena_checkpoint = arena.checkpoint_for_rollback(); for c in &rest[..rest.len() / 2] { rolled_back.insert_change_with_rollback(c.clone(), true, false, &mut rollback); diff --git a/crates/loro-internal/src/oplog/change_store/block_encode.rs b/crates/loro-internal/src/oplog/change_store/block_encode.rs index 48ef273c2..90ce2dd21 100644 --- a/crates/loro-internal/src/oplog/change_store/block_encode.rs +++ b/crates/loro-internal/src/oplog/change_store/block_encode.rs @@ -972,7 +972,9 @@ mod test { println!("Snapshot bytes {:?}", dev_utils::ByteSize(bytes.length())); // assert!(bytes.len() < 30); - let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), true); + let json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv(), true) + .unwrap(); let json_string = serde_json::to_string(&json.changes).unwrap(); println!( "JSON string bytes {:?}", diff --git a/crates/loro-internal/src/oplog/known_history.rs b/crates/loro-internal/src/oplog/known_history.rs index 66e744d3e..583fee19b 100644 --- a/crates/loro-internal/src/oplog/known_history.rs +++ b/crates/loro-internal/src/oplog/known_history.rs @@ -115,7 +115,9 @@ impl OpLog { }; let peer = change.id.peer; let end = end_of(&ends, &peer); - if change.ctr_end() <= end || self.dag.import_deps_before_shallow_root(&change.deps) { + if change.ctr_end() <= end + || self.dag.try_import_deps_before_shallow_root(&change.deps)? + { continue; } diff --git a/crates/loro-internal/src/oplog/loro_dag.rs b/crates/loro-internal/src/oplog/loro_dag.rs index 4c7684d49..eb8c05fe6 100644 --- a/crates/loro-internal/src/oplog/loro_dag.rs +++ b/crates/loro-internal/src/oplog/loro_dag.rs @@ -4,7 +4,9 @@ use crate::id::{Counter, ID}; use crate::span::{HasId, HasLamport}; use crate::sync::Mutex; use crate::version::{shrink_frontiers, Frontiers, ImVersionVector, VersionVector}; -use loro_common::{HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, PeerID}; +use loro_common::{ + HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, LoroError, LoroResult, PeerID, +}; use once_cell::sync::OnceCell; use rle::{HasIndex, HasLength, Mergable, Sliceable}; use rustc_hash::FxHashSet; @@ -58,7 +60,7 @@ pub struct AppDag { #[derive(Debug)] pub(crate) struct AppDagRollback { frontiers: Frontiers, - vv: VersionVector, + vv: Arc, unparsed_vv: VersionVector, shallow_since_frontiers: Frontiers, shallow_root_frontiers_deps: Frontiers, @@ -151,13 +153,13 @@ impl AppDag { &self.shallow_since_frontiers } - pub(crate) fn begin_import_rollback(&mut self) { + pub(crate) fn begin_import_rollback(&mut self, old_vv: Arc) { let old_vv_is_empty = self.vv.is_empty(); let mut rollback = self.import_rollback.lock(); debug_assert!(rollback.is_none()); *rollback = Some(AppDagRollback { frontiers: self.frontiers.clone(), - vv: self.vv.clone(), + vv: old_vv, unparsed_vv: self.unparsed_vv.lock().clone(), shallow_since_frontiers: self.shallow_since_frontiers.clone(), shallow_root_frontiers_deps: self.shallow_root_frontiers_deps.clone(), @@ -204,7 +206,7 @@ impl AppDag { drop(map); self.frontiers = checkpoint.frontiers; - self.vv = checkpoint.vv.clone(); + self.vv = (*checkpoint.vv).clone(); self.shallow_since_frontiers = checkpoint.shallow_since_frontiers; self.shallow_root_frontiers_deps = checkpoint.shallow_root_frontiers_deps; self.shallow_since_vv = checkpoint.shallow_since_vv; @@ -596,9 +598,9 @@ impl AppDag { } } - fn ensure_lazy_load_node(&self, id: ID) { + fn ensure_lazy_load_node(&self, id: ID) -> LoroResult<()> { if self.shallow_since_vv.includes_id(id) { - return; + return Ok(()); } loop { @@ -609,7 +611,7 @@ impl AppDag { unparsed_vv.get(&id.peer).copied().unwrap_or(0) }; if unparsed_end <= id.counter { - return; + return Ok(()); } let last_unparsed_id = ID::new(id.peer, unparsed_end - 1); @@ -617,6 +619,10 @@ impl AppDag { .change_store .get_dag_nodes_that_contains(last_unparsed_id) else { + // An unreadable external block is recorded by ChangeStore; return its + // decode error. A parsable store missing an id promised by unparsed_vv + // is still an impossible internal inconsistency and must fail fast. + self.change_store.corrupt_block_error()?; panic!("unparsed vv don't match with change store. Id:{id} is not in change store") }; @@ -628,7 +634,10 @@ impl AppDag { self.map.lock().len() } - pub(crate) fn set_version_by_fast_snapshot_import(&mut self, v: BatchDecodeInfo) { + pub(crate) fn set_version_by_fast_snapshot_import( + &mut self, + v: BatchDecodeInfo, + ) -> LoroResult<()> { assert!(self.vv.is_empty()); *self.unparsed_vv.lock() = v.vv.clone(); self.vv = v.vv; @@ -637,13 +646,14 @@ impl AppDag { if !f.is_empty() { assert!(f.len() == 1); let id = f.as_single().unwrap(); - let node = self.get(id).unwrap(); + let node = self.try_get(id)?.unwrap(); assert!(node.cnt == id.counter); self.shallow_root_frontiers_deps = node.deps.clone(); } self.shallow_since_frontiers = f; self.shallow_since_vv = ImVersionVector::from_vv(&vv); } + Ok(()) } /// This method is slow and should only be used for debugging and testing. @@ -669,7 +679,7 @@ impl AppDag { let init_counter = self.shallow_since_vv.get(peer).copied().unwrap_or(0); while end_cnt > init_counter { let cnt = end_cnt - 1; - self.ensure_lazy_load_node(ID::new(*peer, cnt)); + self.ensure_lazy_load_node(ID::new(*peer, cnt)).unwrap(); end_cnt = self.unparsed_vv.lock().get(peer).copied().unwrap_or(0); } } @@ -793,13 +803,20 @@ impl AppDag { false } + /// Panics when a dependency block cannot be parsed. Production callers use + /// [`Self::try_import_deps_before_shallow_root`]. + #[allow(dead_code)] pub(crate) fn import_deps_before_shallow_root(&self, deps: &Frontiers) -> bool { + self.try_import_deps_before_shallow_root(deps).unwrap() + } + + pub(crate) fn try_import_deps_before_shallow_root(&self, deps: &Frontiers) -> LoroResult { if self.shallow_since_vv.is_empty() { - return false; + return Ok(false); } if deps.is_empty() { - return true; + return Ok(true); } // Deps equal to the root's own deps describe a change CONCURRENT with @@ -810,12 +827,12 @@ impl AppDag { // be parked as pending and then panic in `calc_unknown_lamport_change`. // Reject it like any other pre-root update. if deps == &self.shallow_root_frontiers_deps { - return true; + return Ok(true); } let shallow_vv = VersionVector::from_im_vv(&self.shallow_since_vv); - if let Some(vv) = self.frontiers_to_vv(deps) { - return !vv.includes_vv(&shallow_vv); + if let Some(vv) = self.try_frontiers_to_vv(deps)? { + return Ok(!vv.includes_vv(&shallow_vv)); } // Import only needs to reject updates whose causal source is older than @@ -826,20 +843,20 @@ impl AppDag { .iter() .any(|id| self.shallow_since_frontiers.contains(&id)) { - return false; + return Ok(false); } - deps.iter().any(|id| self.shallow_since_vv.includes_id(id)) + Ok(deps.iter().any(|id| self.shallow_since_vv.includes_id(id))) } /// Travel the ancestors of the given id, and call the callback for each node /// /// It will travel the ancestors in the reverse order (from the greatest lamport to the smallest) - pub(crate) fn travel_ancestors( + pub(crate) fn try_travel_ancestors( &self, id: ID, f: &mut dyn FnMut(&AppDagNode) -> ControlFlow<()>, - ) { + ) -> LoroResult<()> { struct PendingNode(AppDagNode); impl PartialEq for PendingNode { fn eq(&self, other: &Self) -> bool { @@ -863,14 +880,16 @@ impl AppDag { let mut visited = FxHashSet::default(); let mut pending: BinaryHeap = BinaryHeap::new(); - pending.push(PendingNode(self.get(id).unwrap())); + pending.push(PendingNode( + self.try_get(id)?.ok_or(LoroError::FrontiersNotFound(id))?, + )); while let Some(PendingNode(node)) = pending.pop() { if f(&node).is_break() { break; } for dep in node.deps.iter() { - let Some(dep_node) = self.get(dep) else { + let Some(dep_node) = self.try_get(dep)? else { continue; }; if visited.contains(&dep_node.id_start()) { @@ -881,6 +900,7 @@ impl AppDag { pending.push(PendingNode(dep_node)); } } + Ok(()) } pub(crate) fn update_version_on_new_local_op( @@ -1066,24 +1086,28 @@ impl Dag for AppDag { } fn get(&self, id: ID) -> Option { - self.ensure_lazy_load_node(id); + self.try_get(id).unwrap() + } + + fn try_get(&self, id: ID) -> LoroResult> { + self.ensure_lazy_load_node(id)?; let binding = self.map.lock(); if let Some(x) = binding.range(..=id).next_back() { if x.1.contains_id(id) { // PERF: do we need to optimize clone like this? // by adding another layer of Arc? - return Some(x.1.clone()); + return Ok(Some(x.1.clone())); } } if let Some(node) = &self.pending_txn_node { if node.peer == id.peer && node.cnt <= id.counter { assert!(node.cnt + node.len as Counter > id.counter); - return Some(node.clone()); + return Ok(Some(node.clone())); } } - None + Ok(None) } fn vv(&self) -> &VersionVector { @@ -1108,6 +1132,13 @@ impl AppDag { } pub(crate) fn ensure_vv_for(&self, target_node: &AppDagNode) -> ImVersionVector { + self.try_ensure_vv_for(target_node).unwrap() + } + + pub(crate) fn try_ensure_vv_for( + &self, + target_node: &AppDagNode, + ) -> LoroResult { if target_node.vv.get().is_none() { // Iterative DFS. When the DAG contains a diamond, a dep can end // up on the stack multiple times (once for each ancestor that @@ -1129,7 +1160,7 @@ impl AppDag { } else { let mut all_deps_processed = true; for id in top_node.deps.iter() { - let Some(node) = self.get(id) else { + let Some(node) = self.try_get(id)? else { if self.shallow_since_vv.includes_id(id) { continue; } @@ -1151,7 +1182,7 @@ impl AppDag { } for id in top_node.deps.iter() { - let Some(node) = self.get(id) else { + let Some(node) = self.try_get(id)? else { if self.shallow_since_vv.includes_id(id) { ans_vv.extend_to_include_vv(self.shallow_since_vv.iter()); continue; @@ -1177,7 +1208,7 @@ impl AppDag { } } - target_node.vv.get().unwrap().clone() + Ok(target_node.vv.get().unwrap().clone()) } /// Compare the causal order of two versions. @@ -1193,44 +1224,63 @@ impl AppDag { } pub fn get_lamport(&self, id: &ID) -> Option { - self.get(*id).and_then(|node| { + self.try_get_lamport(id).unwrap() + } + + pub fn try_get_lamport(&self, id: &ID) -> LoroResult> { + Ok(self.try_get(*id)?.and_then(|node| { assert!(id.counter >= node.cnt); if node.cnt + node.len as Counter > id.counter { Some(node.lamport + (id.counter - node.cnt) as Lamport) } else { None } - }) + })) } pub fn get_change_lamport_from_deps(&self, deps: &Frontiers) -> Option { + self.try_get_change_lamport_from_deps(deps).unwrap() + } + + pub fn try_get_change_lamport_from_deps( + &self, + deps: &Frontiers, + ) -> LoroResult> { let mut lamport = 0; for id in deps.iter() { - let x = self.get_lamport(&id)?; + let Some(x) = self.try_get_lamport(&id)? else { + return Ok(None); + }; lamport = lamport.max(x + 1); } - Some(lamport) + Ok(Some(lamport)) } /// Convert a frontiers to a version vector /// /// If the frontiers version is not found in the dag, return None pub fn frontiers_to_vv(&self, frontiers: &Frontiers) -> Option { + self.try_frontiers_to_vv(frontiers).unwrap() + } + + pub fn try_frontiers_to_vv(&self, frontiers: &Frontiers) -> LoroResult> { if frontiers == &self.shallow_root_frontiers_deps { let vv = VersionVector::from_im_vv(&self.shallow_since_vv); - return Some(vv); + return Ok(Some(vv)); } let mut vv: VersionVector = Default::default(); for id in frontiers.iter() { - let x = self.get(id)?; - let target_vv = self.ensure_vv_for(&x); + let Some(x) = self.try_get(id)? else { + return Ok(None); + }; + let target_vv = self.try_ensure_vv_for(&x)?; vv.extend_to_include_vv(target_vv.iter()); vv.extend_to_include_last_id(id); } - Some(vv) + Ok(Some(vv)) } #[allow(unused)] @@ -1294,8 +1344,12 @@ impl AppDag { } pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers { + self.try_vv_to_frontiers(vv).unwrap() + } + + pub fn try_vv_to_frontiers(&self, vv: &VersionVector) -> LoroResult { if vv.is_empty() { - return Default::default(); + return Ok(Default::default()); } let this = vv; @@ -1318,10 +1372,10 @@ impl AppDag { .collect(); if last_ids.is_empty() { - return self.shallow_since_frontiers.clone(); + return Ok(self.shallow_since_frontiers.clone()); } - shrink_frontiers(&last_ids, self).unwrap() + crate::version::try_shrink_frontiers(&last_ids, self) } pub(crate) fn frontiers_to_next_lamport(&self, frontiers: &Frontiers) -> Lamport { @@ -1376,17 +1430,56 @@ impl AppDag { a: &Frontiers, b: &Frontiers, ) -> Result, FrontiersNotIncluded> { - let a = self.frontiers_to_vv(a).ok_or(FrontiersNotIncluded)?; - let b = self.frontiers_to_vv(b).ok_or(FrontiersNotIncluded)?; + let a = self + .try_frontiers_to_vv(a) + .map_err(FrontiersNotIncluded::unreadable)? + .ok_or(FrontiersNotIncluded)?; + let b = self + .try_frontiers_to_vv(b) + .map_err(FrontiersNotIncluded::unreadable)? + .ok_or(FrontiersNotIncluded)?; Ok(a.partial_cmp(&b)) } } +/// Frontiers were not found, or comparing them had to read history that cannot +/// be parsed. +/// +/// Construct an absent-frontiers error with the [`FrontiersNotIncluded`] +/// constant. That construction still compiles, but an exhaustive `match` of +/// `Err(FrontiersNotIncluded)` does not (`E0004`): the struct has a private +/// field, so a `history_error: Some(_)` pattern cannot be written from another +/// crate (`E0451`). Downstream code must use `Err(_)`. An unreadable-history +/// error is not equal to the [`FrontiersNotIncluded`] constant. #[derive(Debug, PartialEq, Eq)] -pub struct FrontiersNotIncluded; +pub struct FrontiersNotIncluded { + history_error: Option>, +} + +/// Absent frontiers. Not equal to an error produced from unreadable history. +/// +/// See [`FrontiersNotIncluded`] for why an exhaustive match of this value does +/// not compile. +#[allow(non_upper_case_globals)] +pub const FrontiersNotIncluded: FrontiersNotIncluded = FrontiersNotIncluded { + history_error: None, +}; + +impl FrontiersNotIncluded { + pub(crate) fn unreadable(error: LoroError) -> Self { + Self { + history_error: Some(error.to_string().into_boxed_str()), + } + } +} + impl Display for FrontiersNotIncluded { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("The given Frontiers are not included by the doc") + f.write_str( + self.history_error + .as_deref() + .unwrap_or("The given Frontiers are not included by the doc"), + ) } } @@ -1426,6 +1519,15 @@ mod ensure_vv_for_tests { dag } + #[test] + #[should_panic(expected = "unparsed vv don't match with change store")] + fn a_missing_node_without_a_parse_failure_remains_an_internal_invariant() { + let store = ChangeStore::new_mem(&SharedArena::new(), Arc::new(AtomicI64::new(0))); + let dag = AppDag::new(store); + dag.unparsed_vv.lock().insert(1, 1); + let _ = dag.try_get(ID::new(1, 0)); + } + /// Regression for loro-dev/loro#929: when computing the vv for a node /// whose DAG fan-in contains a shared ancestor reached through multiple /// paths, the iterative DFS used to push the shared ancestor onto the diff --git a/crates/loro-internal/src/tests/import_atomicity.rs b/crates/loro-internal/src/tests/import_atomicity.rs index 9252a56ef..47759a7ff 100644 --- a/crates/loro-internal/src/tests/import_atomicity.rs +++ b/crates/loro-internal/src/tests/import_atomicity.rs @@ -84,7 +84,9 @@ fn make_json_list_update_with_four_ops(peer: u64) -> (LoroDoc, JsonSchema) { list.insert_with_txn(&mut txn, 1, "tail".into()).unwrap(); txn.commit().unwrap(); - let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false); + let json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv(), false) + .unwrap(); assert_eq!(json.changes.len(), 1); assert_eq!(json.changes[0].ops.len(), 4); (doc, json) @@ -752,7 +754,7 @@ fn binary_update_bypassing_validation(doc: &LoroDoc, json: serde_json::Value) -> let mut oplog = carrier.oplog().lock(); let changes = crate::encoding::json_schema::decode_json_changes(json, &oplog.arena).unwrap(); - crate::encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + crate::encoding::apply_decoded_changes_to_oplog(&mut oplog, changes).unwrap(); } carrier .export(ExportMode::updates(&doc.oplog_vv())) diff --git a/crates/loro-internal/src/undo.rs b/crates/loro-internal/src/undo.rs index 20aea7cc4..f6e61b452 100644 --- a/crates/loro-internal/src/undo.rs +++ b/crates/loro-internal/src/undo.rs @@ -959,8 +959,7 @@ impl UndoManager { return Err(e); } Err(e) => { - get_stack(&mut self.inner.lock().borrow_mut()) - .push(span.span, span.meta); + get_stack(&mut self.inner.lock().borrow_mut()).push(span.span, span.meta); return Err(e); } }; diff --git a/crates/loro-internal/src/version.rs b/crates/loro-internal/src/version.rs index 3efbe1498..3c3edb362 100644 --- a/crates/loro-internal/src/version.rs +++ b/crates/loro-internal/src/version.rs @@ -994,6 +994,16 @@ impl VersionVector { /// Use minimal set of ids to represent the frontiers #[tracing::instrument(skip(dag))] pub fn shrink_frontiers(last_ids: &Frontiers, dag: &AppDag) -> Result { + try_shrink_frontiers(last_ids, dag).map_err(|err| match err { + loro_common::LoroError::FrontiersNotFound(id) => id, + err => panic!("{err}"), + }) +} + +pub(crate) fn try_shrink_frontiers( + last_ids: &Frontiers, + dag: &AppDag, +) -> loro_common::LoroResult { // it only keep the ids of ops that are concurrent to each other if last_ids.len() <= 1 { @@ -1010,8 +1020,8 @@ pub fn shrink_frontiers(last_ids: &Frontiers, dag: &AppDag) -> Result Result Result VersionVector { - VersionVector(self.doc.state_vv()) + pub fn version(&self) -> JsResult { + Ok(VersionVector(self.doc.try_state_vv()?)) } /// The doc only contains the history since this version @@ -1696,11 +1696,11 @@ impl LoroDoc { temp_end_vv = Some(js_to_version_vector(end_vv)?); json_end_vv = &temp_end_vv.as_ref().unwrap().0; } - let json_schema = self.doc.export_json_updates( + let json_schema = self.doc.try_export_json_updates( json_start_vv, json_end_vv, with_peer_compression.unwrap_or(true), - ); + )?; loro_json_schema_to_js_json_schema(json_schema) } @@ -2037,7 +2037,7 @@ impl LoroDoc { let borrow_mut = &self.doc; let oplog = borrow_mut.oplog().lock(); let change = oplog - .get_change_at(id) + .try_get_change_at(id)? .ok_or_else(|| JsError::new(&format!("Change {:?} not found", id)))?; let change = ChangeMeta { lamport: change.lamport(), @@ -2070,7 +2070,8 @@ impl LoroDoc { let peer_id = peer_id .parse() .map_err(|_| JsValue::from_str(ID_CONVERT_ERROR))?; - let Some(change) = oplog.get_change_with_lamport_lte(peer_id, lamport) else { + let change = oplog.try_get_change_with_lamport_lte(peer_id, lamport)?; + let Some(change) = change else { return Ok(JsValue::UNDEFINED.into()); }; @@ -2104,7 +2105,7 @@ impl LoroDoc { serde_wasm_bindgen::Serializer::new().serialize_large_number_types_as_bigints(true); let change = oplog - .get_remote_change_at(id) + .try_get_remote_change_at(id)? .ok_or_else(|| JsError::new(&format!("Change {:?} not found", id)))?; let ops = change .ops() @@ -2132,11 +2133,8 @@ impl LoroDoc { #[wasm_bindgen(js_name = "frontiersToVV")] pub fn frontiers_to_vv(&self, frontiers: Vec) -> JsResult { let frontiers = ids_to_frontiers(frontiers)?; - let borrow_mut = &self.doc; - let oplog = borrow_mut.oplog().lock(); - oplog - .dag() - .frontiers_to_vv(&frontiers) + self.doc + .try_frontiers_to_vv(&frontiers)? .map(VersionVector) .ok_or_else(|| JsError::new("Frontiers not found").into()) } @@ -2155,7 +2153,7 @@ impl LoroDoc { /// ``` #[wasm_bindgen(js_name = "vvToFrontiers")] pub fn vv_to_frontiers(&self, vv: &VersionVector) -> JsResult { - let f = self.doc.oplog().lock().dag().vv_to_frontiers(&vv.0); + let f = self.doc.try_vv_to_frontiers(&vv.0)?; frontiers_to_ids(&f) } @@ -2216,20 +2214,20 @@ impl LoroDoc { /// } /// ``` pub fn getCursorPos(&self, cursor: &Cursor) -> JsResult { - let cursor = self.doc.query_pos(&cursor.pos); - let ans = match cursor { + let ans = match self.doc.try_get_cursor_pos(&cursor.pos) { Ok(ans) => ans, - Err( - CannotFindRelativePosition::ContainerDeleted - | CannotFindRelativePosition::IdNotFound, - ) => return Ok(JsValue::UNDEFINED.into()), - Err(CannotFindRelativePosition::HistoryCleared) => { + Err(loro_internal::LoroError::NotFoundError(msg)) + if msg.contains("history is cleared") => + { console_warn!( - "Cannot find cursor position of {:?} because the related history is cleared.", - cursor + "Cannot find cursor position because the related history is cleared: {msg}" ); return Ok(JsValue::UNDEFINED.into()); } + Err(loro_internal::LoroError::NotFoundError(_)) => { + return Ok(JsValue::UNDEFINED.into()); + } + Err(err) => return Err(err.into()), }; let obj = Object::new(); @@ -2266,7 +2264,7 @@ impl LoroDoc { let id = js_id_to_id(id)?; Ok(self .doc - .get_changed_containers_in(id, len) + .try_get_changed_containers_in(id, len)? .into_iter() .map(|cid| { let v: JsValue = (&cid).into(); diff --git a/crates/loro-wasm/tests/fixtures/unparsable-body.snapshot.bin b/crates/loro-wasm/tests/fixtures/unparsable-body.snapshot.bin new file mode 100644 index 000000000..6db127385 Binary files /dev/null and b/crates/loro-wasm/tests/fixtures/unparsable-body.snapshot.bin differ diff --git a/crates/loro-wasm/tests/fixtures/unparsable-history.snapshot.bin b/crates/loro-wasm/tests/fixtures/unparsable-history.snapshot.bin new file mode 100644 index 000000000..6588f0079 Binary files /dev/null and b/crates/loro-wasm/tests/fixtures/unparsable-history.snapshot.bin differ diff --git a/crates/loro-wasm/tests/unparsable_history.test.ts b/crates/loro-wasm/tests/unparsable_history.test.ts new file mode 100644 index 000000000..661384acd --- /dev/null +++ b/crates/loro-wasm/tests/unparsable_history.test.ts @@ -0,0 +1,102 @@ +import { readFileSync } from "node:fs"; +import { expect, it } from "vitest"; +import { LoroDoc, VersionVector, type Frontiers } from "../bundler/index"; + +// Generated by the public Rust regression with LORO_HISTORY_FIXTURE set to this +// absolute path. The first peer-1 block is halved and both checksums are repaired. +const snapshot = readFileSync( + new URL("./fixtures/unparsable-history.snapshot.bin", import.meta.url), +); +const mid: Frontiers = [{ peer: "1", counter: 1 }]; +const remote = new LoroDoc(); +remote.setPeerId(9); +remote.getText("text").insert(0, "concurrent"); +const updates = remote.export({ mode: "update" }); +const json = remote.exportJsonUpdates(); + +const readers: Record unknown> = { + checkout: (doc) => doc.checkout(mid), + diff: (doc) => doc.diff(mid, doc.oplogFrontiers()), + revertTo: (doc) => doc.revertTo(mid), + import: (doc) => doc.import(updates), + importBatch: (doc) => doc.importBatch([updates, updates]), + importJsonUpdates: (doc) => doc.importJsonUpdates(json), + forkAt: (doc) => doc.forkAt(mid), + findIdSpansBetween: (doc) => + doc.findIdSpansBetween(mid, doc.oplogFrontiers()), + frontiersToVV: (doc) => doc.frontiersToVV(mid), + vvToFrontiers: (doc) => { + const vv = new VersionVector(doc.oplogVersion().toJSON()); + vv.setEnd({ peer: "1", counter: 2 }); + return doc.vvToFrontiers(vv); + }, + travelChangeAncestors: (doc) => doc.travelChangeAncestors(mid, () => true), + getChangeAt: (doc) => doc.getChangeAt(mid[0]), + getChangeAtLamport: (doc) => doc.getChangeAtLamport("1", 1), + getOpsInChange: (doc) => doc.getOpsInChange(mid[0]), + getChangedContainersIn: (doc) => doc.getChangedContainersIn(mid[0], 1), + cmpFrontiers: (doc) => doc.cmpFrontiers(mid, doc.oplogFrontiers()), + exportUpdates: (doc) => doc.export({ mode: "update" }), + exportUpdatesInRange: (doc) => + doc.export({ + mode: "updates-in-range", + spans: [{ id: mid[0], len: 1 }], + }), + exportShallow: (doc) => + doc.export({ mode: "shallow-snapshot", frontiers: mid }), +}; + +for (const [name, read] of Object.entries(readers)) { + it(`${name} rejects its first unreadable block without trapping WASM`, () => { + const doc = new LoroDoc(); + doc.import(snapshot); + const value = doc.toJSON(); + const frontiers = doc.frontiers(); + const vv = doc.oplogVersion().toJSON(); + let events = 0; + doc.subscribe(() => events++); + expect(() => read(doc)).toThrow(/cannot parse change block/); + expect(doc.toJSON()).toEqual(value); + expect(doc.frontiers()).toEqual(frontiers); + expect(doc.oplogVersion().toJSON()).toEqual(vv); + expect(doc.isDetached()).toBe(false); + expect(events).toBe(0); + expect(() => doc.checkout(frontiers)).toThrow(/cannot parse change block/); + // A fresh document in the same module still works: no WASM trap occurred. + const healthy = new LoroDoc(); + healthy.getText("text").insert(0, "usable"); + expect(healthy.toJSON()).toEqual({ text: "usable" }); + }); +} + +it("rolls back a map import after reading only the old block headers", () => { + // Generated by the Rust register-only regression with LORO_BODY_FIXTURE set + // to this path: XOR byte -3 in the first peer-1 block, repair both checksums. + const bodyCorrupted = readFileSync( + new URL("./fixtures/unparsable-body.snapshot.bin", import.meta.url), + ); + const doc = new LoroDoc(); + doc.import(bodyCorrupted); + expect(doc.frontiersToVV([{ peer: "1", counter: 0 }])).toBeDefined(); + expect(doc.frontiersToVV([{ peer: "2", counter: 0 }])).toBeDefined(); + const value = doc.toJSON(); + const frontiers = doc.frontiers(); + const vv = doc.oplogVersion().toJSON(); + const mapRemote = new LoroDoc(); + mapRemote.setPeerId(10); + mapRemote.getMap("map").set("key", -1); + const mapUpdates = mapRemote.export({ mode: "update" }); + let events = 0; + doc.subscribe(() => events++); + expect(() => doc.import(mapUpdates)).toThrow(/cannot parse change block/); + expect(doc.toJSON()).toEqual(value); + expect(doc.frontiers()).toEqual(frontiers); + expect(doc.oplogFrontiers()).toEqual(frontiers); + expect(doc.oplogVersion().toJSON()).toEqual(vv); + expect(doc.isDetached()).toBe(false); + expect(events).toBe(0); + expect(() => doc.checkout(frontiers)).toThrow(/cannot parse change block/); + const healthy = new LoroDoc(); + healthy.getMap("map").set("key", "usable"); + expect(healthy.toJSON()).toEqual({ map: { key: "usable" } }); +}); diff --git a/crates/loro/Cargo.toml b/crates/loro/Cargo.toml index 9fa1b0579..7389a90cf 100644 --- a/crates/loro/Cargo.toml +++ b/crates/loro/Cargo.toml @@ -26,6 +26,7 @@ tracing = { workspace = true } rustc-hash = { workspace = true } [dev-dependencies] +xxhash-rust = { workspace = true } serde_json = { version = "1.0.87", features = ["float_roundtrip"] } anyhow = "1.0.83" ctor = "0.2" diff --git a/crates/loro/examples/moon_golden_gen.rs b/crates/loro/examples/moon_golden_gen.rs index ba9a0f58e..a8e7ab900 100644 --- a/crates/loro/examples/moon_golden_gen.rs +++ b/crates/loro/examples/moon_golden_gen.rs @@ -398,7 +398,7 @@ fn main() -> anyhow::Result<()> { })?; std::fs::write(out_dir.join("updates.blob"), &updates_blob)?; - let updates_schema = doc.export_json_updates(&start, &end); + let updates_schema = doc.export_json_updates(&start, &end).unwrap(); let updates_json = serde_json::to_value(&updates_schema)?; write_json(&out_dir.join("updates.json"), &updates_json)?; diff --git a/crates/loro/examples/moon_jsonschema_fuzz.rs b/crates/loro/examples/moon_jsonschema_fuzz.rs index 17d6edced..989a9d7a7 100644 --- a/crates/loro/examples/moon_jsonschema_fuzz.rs +++ b/crates/loro/examples/moon_jsonschema_fuzz.rs @@ -586,7 +586,7 @@ fn main() -> anyhow::Result<()> { expected_doc.import(&rust_updates_blob)?; let expected = expected_doc.get_deep_value().to_json_value(); - let schema = doc.export_json_updates(&start_vv, &end); + let schema = doc.export_json_updates(&start_vv, &end).unwrap(); let json = serde_json::to_string(&schema)?; let out_blob = run_encode_jsonschema(&node_bin, &cli_js, &json)?; diff --git a/crates/loro/src/lib.rs b/crates/loro/src/lib.rs index 4ff8e9779..7d7f57355 100644 --- a/crates/loro/src/lib.rs +++ b/crates/loro/src/lib.rs @@ -186,6 +186,11 @@ impl LoroDoc { Some(ChangeMeta::from_change(&change)) } + /// Reads change metadata, reporting unreadable history instead of an absent change. + pub fn try_get_change(&self, id: ID) -> LoroResult> { + self.doc.try_get_change(id) + } + /// Decodes the metadata for an imported blob from the provided bytes. /// /// # Example @@ -730,7 +735,7 @@ impl LoroDoc { /// let a = LoroDoc::new(); /// a.get_text("t").insert(0, "hi").unwrap(); /// a.commit(); - /// let json = a.export_json_updates(&VersionVector::default(), &a.oplog_vv()); + /// let json = a.export_json_updates(&VersionVector::default(), &a.oplog_vv()).unwrap(); /// /// let b = LoroDoc::new(); /// b.import_json_updates(json).unwrap(); @@ -752,15 +757,28 @@ impl LoroDoc { /// let doc = LoroDoc::new(); /// let start = VersionVector::default(); /// let end = doc.oplog_vv(); - /// let json = doc.export_json_updates(&start, &end); + /// let json = doc.export_json_updates(&start, &end).unwrap(); /// ``` + /// + /// Returns [`LoroError::DecodeError`] when the range has to read a change + /// block that cannot be parsed. #[inline] pub fn export_json_updates( &self, start_vv: &VersionVector, end_vv: &VersionVector, - ) -> JsonSchema { - self.doc.export_json_updates(start_vv, end_vv, true) + ) -> LoroResult { + self.try_export_json_updates(start_vv, end_vv) + } + + /// [`export_json_updates`](Self::export_json_updates). + #[inline] + pub fn try_export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + ) -> LoroResult { + self.doc.try_export_json_updates(start_vv, end_vv, true) } /// Export the current state with json-string format of the document, without peer compression. @@ -774,15 +792,15 @@ impl LoroDoc { /// let doc = LoroDoc::new(); /// let start = VersionVector::default(); /// let end = doc.oplog_vv(); - /// let json = doc.export_json_updates_without_peer_compression(&start, &end); + /// let json = doc.export_json_updates_without_peer_compression(&start, &end).unwrap(); /// ``` #[inline] pub fn export_json_updates_without_peer_compression( &self, start_vv: &VersionVector, end_vv: &VersionVector, - ) -> JsonSchema { - self.doc.export_json_updates(start_vv, end_vv, false) + ) -> LoroResult { + self.doc.try_export_json_updates(start_vv, end_vv, false) } /// Exports changes within the specified ID span to JSON schema format. @@ -862,6 +880,21 @@ impl LoroDoc { self.doc.vv_to_frontiers(vv) } + /// Converts frontiers to a version vector, reporting unreadable history. + pub fn try_frontiers_to_vv(&self, frontiers: &Frontiers) -> LoroResult> { + self.doc.try_frontiers_to_vv(frontiers) + } + + /// Converts a version vector to frontiers, reporting unreadable history. + pub fn try_vv_to_frontiers(&self, vv: &VersionVector) -> LoroResult { + self.doc.try_vv_to_frontiers(vv) + } + + /// Removes redundant frontiers, reporting unreadable history or an absent id. + pub fn try_minimize_frontiers(&self, frontiers: &Frontiers) -> LoroResult { + self.doc.try_minimize_frontiers(frontiers) + } + /// Access the `OpLog`. /// /// NOTE: The API in `OpLog` is unstable. Keep the closure short; avoid calling methods @@ -888,10 +921,22 @@ impl LoroDoc { self.doc.oplog_vv() } - /// Get the `VersionVector` version of `DocState` + /// Get the `VersionVector` version of `DocState`. + /// + /// When the state is at the op-log frontiers this clones the cached op-log + /// version vector and does not parse change blocks. Otherwise it walks the + /// frontiers. A change block that cannot be parsed is [`LoroError::DecodeError`]. + /// Frontiers that are not in the dag are [`LoroError::FrontiersNotFound`] when + /// an id can be named, not an empty version vector. + #[inline] + pub fn state_vv(&self) -> LoroResult { + self.try_state_vv() + } + + /// [`state_vv`](Self::state_vv). #[inline] - pub fn state_vv(&self) -> VersionVector { - self.doc.state_vv() + pub fn try_state_vv(&self) -> LoroResult { + self.doc.try_state_vv() } /// The doc only contains the history since this version @@ -1236,6 +1281,16 @@ impl LoroDoc { self.doc.query_pos(cursor) } + /// [`get_cursor_pos`](Self::get_cursor_pos), preserving a decode error. + /// + /// [`get_cursor_pos`](Self::get_cursor_pos) maps an unreadable change block to + /// [`CannotFindRelativePosition::HistoryUnreadable`]. This method returns the + /// [`LoroError::DecodeError`] itself. + #[inline] + pub fn try_get_cursor_pos(&self, cursor: &Cursor) -> LoroResult { + self.doc.try_get_cursor_pos(cursor) + } + /// Get the inner LoroDoc ref. #[inline] pub fn inner(&self) -> &InnerLoroDoc { @@ -1443,6 +1498,24 @@ impl LoroDoc { self.doc.find_id_spans_between(from, to) } + /// Finds operation spans between versions, reporting unreadable history. + pub fn try_find_id_spans_between( + &self, + from: &Frontiers, + to: &Frontiers, + ) -> LoroResult { + self.doc.try_find_id_spans_between(from, to) + } + + /// Gets modified container IDs, reporting unreadable history. + pub fn try_get_changed_containers_in( + &self, + id: ID, + len: usize, + ) -> LoroResult> { + self.doc.try_get_changed_containers_in(id, len) + } + /// Revert the current document state back to the target version /// /// Internally, it will generate a series of local operations that can revert the diff --git a/crates/loro/tests/commit_message_test.rs b/crates/loro/tests/commit_message_test.rs index ecf16899b..0837abdf6 100644 --- a/crates/loro/tests/commit_message_test.rs +++ b/crates/loro/tests/commit_message_test.rs @@ -191,7 +191,7 @@ fn test_commit_message_json_updates() { let start_vv = VersionVector::new(); let end_vv = doc1.oplog_vv(); - let json_updates = doc1.export_json_updates(&start_vv, &end_vv); + let json_updates = doc1.export_json_updates(&start_vv, &end_vv).unwrap(); let doc2 = LoroDoc::new(); doc2.import_json_updates(json_updates).unwrap(); diff --git a/crates/loro/tests/contracts/counter.rs b/crates/loro/tests/contracts/counter.rs index e33d862f5..b428773b9 100644 --- a/crates/loro/tests/contracts/counter.rs +++ b/crates/loro/tests/contracts/counter.rs @@ -90,7 +90,9 @@ fn counter_events_json_updates_and_deletion_follow_contract() -> LoroResult<()> &[("counter-local".to_string(), 7.0)] ); - let json_updates = doc.export_json_updates(&VersionVector::default(), &doc.oplog_vv()); + let json_updates = doc + .export_json_updates(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); let imported = LoroDoc::new(); imported.import_json_updates(json_updates)?; assert_eq!( @@ -168,7 +170,9 @@ fn counters_nested_in_tree_meta_and_movable_list_survive_diff_and_snapshot() -> restored.apply_diff(reverse)?; assert_eq!(restored.get_deep_value().to_json_value(), json_v1); - let json_updates = doc.export_json_updates(&VersionVector::default(), &doc.oplog_vv()); + let json_updates = doc + .export_json_updates(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); let imported = LoroDoc::new(); imported.import_json_updates(json_updates)?; assert_eq!( diff --git a/crates/loro/tests/contracts/doc_export.rs b/crates/loro/tests/contracts/doc_export.rs index ec8657c2e..d1ebfa58c 100644 --- a/crates/loro/tests/contracts/doc_export.rs +++ b/crates/loro/tests/contracts/doc_export.rs @@ -216,8 +216,10 @@ fn commit_metadata_empty_commit_and_json_updates_roundtrip_follow_contract() -> let start = VersionVector::default(); let end = doc.oplog_vv(); - let compressed = doc.export_json_updates(&start, &end); - let uncompressed = doc.export_json_updates_without_peer_compression(&start, &end); + let compressed = doc.export_json_updates(&start, &end).unwrap(); + let uncompressed = doc + .export_json_updates_without_peer_compression(&start, &end) + .unwrap(); assert!(compressed.peers.is_some()); assert!(uncompressed.peers.is_none()); diff --git a/crates/loro/tests/contracts/doc_lifecycle.rs b/crates/loro/tests/contracts/doc_lifecycle.rs index 1be34389c..a5ca1101b 100644 --- a/crates/loro/tests/contracts/doc_lifecycle.rs +++ b/crates/loro/tests/contracts/doc_lifecycle.rs @@ -308,8 +308,10 @@ fn storage_blobs_json_updates_and_batch_import_follow_contract() -> anyhow::Resu let start = VersionVector::default(); let end = doc.oplog_vv(); - let compressed = doc.export_json_updates(&start, &end); - let uncompressed = doc.export_json_updates_without_peer_compression(&start, &end); + let compressed = doc.export_json_updates(&start, &end).unwrap(); + let uncompressed = doc + .export_json_updates_without_peer_compression(&start, &end) + .unwrap(); assert!(compressed.peers.is_some()); assert!(uncompressed.peers.is_none()); diff --git a/crates/loro/tests/contracts/storage_encoding.rs b/crates/loro/tests/contracts/storage_encoding.rs index c8cd989c9..b943de14b 100644 --- a/crates/loro/tests/contracts/storage_encoding.rs +++ b/crates/loro/tests/contracts/storage_encoding.rs @@ -348,8 +348,10 @@ fn storage_blobs_and_json_schema_roundtrip_state_and_metadata() -> anyhow::Resul let start = VersionVector::default(); let end = source.oplog_vv(); - let compressed = source.export_json_updates(&start, &end); - let uncompressed = source.export_json_updates_without_peer_compression(&start, &end); + let compressed = source.export_json_updates(&start, &end).unwrap(); + let uncompressed = source + .export_json_updates_without_peer_compression(&start, &end) + .unwrap(); assert!(compressed.peers.is_some()); assert!(uncompressed.peers.is_none()); diff --git a/crates/loro/tests/contracts/sync_import.rs b/crates/loro/tests/contracts/sync_import.rs index 4d10b947f..062057e29 100644 --- a/crates/loro/tests/contracts/sync_import.rs +++ b/crates/loro/tests/contracts/sync_import.rs @@ -251,7 +251,9 @@ fn update_with_out_of_bounds_list_insert(peer: u64) -> LoroResult> { doc.get_list("list").insert(1, "tail")?; doc.commit(); - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let last_op = json .changes .last_mut() diff --git a/crates/loro/tests/contracts/value_conversion.rs b/crates/loro/tests/contracts/value_conversion.rs index 7e599a828..5c58d86e9 100644 --- a/crates/loro/tests/contracts/value_conversion.rs +++ b/crates/loro/tests/contracts/value_conversion.rs @@ -285,8 +285,10 @@ fn json_schema_roundtrip_keeps_value_types_and_imports_back_state() -> anyhow::R let start = VersionVector::default(); let end = doc.oplog_vv(); - let compressed = doc.export_json_updates(&start, &end); - let uncompressed = doc.export_json_updates_without_peer_compression(&start, &end); + let compressed = doc.export_json_updates(&start, &end).unwrap(); + let uncompressed = doc + .export_json_updates_without_peer_compression(&start, &end) + .unwrap(); let compressed_json = serde_json::to_string(&compressed)?; let uncompressed_json = serde_json::to_string(&uncompressed)?; diff --git a/crates/loro/tests/contracts/value_diff.rs b/crates/loro/tests/contracts/value_diff.rs index 6db252933..4cb3bec80 100644 --- a/crates/loro/tests/contracts/value_diff.rs +++ b/crates/loro/tests/contracts/value_diff.rs @@ -211,8 +211,10 @@ fn mixed_state_roundtrips_through_updates_json_and_snapshots() -> anyhow::Result let start = VersionVector::default(); let end = doc.oplog_vv(); - let json_updates = doc.export_json_updates(&start, &end); - let json_updates_no_peers = doc.export_json_updates_without_peer_compression(&start, &end); + let json_updates = doc.export_json_updates(&start, &end).unwrap(); + let json_updates_no_peers = doc + .export_json_updates_without_peer_compression(&start, &end) + .unwrap(); assert!(json_updates.peers.is_some()); assert!(json_updates_no_peers.peers.is_none()); diff --git a/crates/loro/tests/contracts/value_encoding.rs b/crates/loro/tests/contracts/value_encoding.rs index cd6a99f59..de9adc257 100644 --- a/crates/loro/tests/contracts/value_encoding.rs +++ b/crates/loro/tests/contracts/value_encoding.rs @@ -313,8 +313,10 @@ fn json_updates_roundtrip_nested_values_and_peer_compression() -> anyhow::Result let start = VersionVector::default(); let end = doc.oplog_vv(); - let compressed = doc.export_json_updates(&start, &end); - let uncompressed = doc.export_json_updates_without_peer_compression(&start, &end); + let compressed = doc.export_json_updates(&start, &end).unwrap(); + let uncompressed = doc + .export_json_updates_without_peer_compression(&start, &end) + .unwrap(); assert_eq!(compressed.schema_version, 1); assert_eq!(uncompressed.schema_version, 1); @@ -391,7 +393,8 @@ fn import_json_updates_accepts_reordered_op_fields() -> anyhow::Result<()> { doc.commit(); let json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); let value = serde_json::to_value(json)?; let change = &value["changes"][0]; let op = &change["ops"][0]; @@ -428,7 +431,8 @@ fn import_json_updates_rejects_unsupported_schema_version() -> anyhow::Result<() doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); json.schema_version = 2; let err = LoroDoc::new().import_json_updates(json).unwrap_err(); @@ -470,14 +474,18 @@ fn export_json_updates_clamps_negative_version_ranges() -> anyhow::Result<()> { let mut negative_start = VersionVector::new(); negative_start.insert(doc.peer_id(), i32::MIN); - let json = doc.export_json_updates(&negative_start, &doc.oplog_vv()); + let json = doc + .export_json_updates(&negative_start, &doc.oplog_vv()) + .unwrap(); let restored = LoroDoc::new(); restored.import_json_updates(json)?; assert_eq!(restored.get_text("text").to_string(), "a"); let mut negative_end = VersionVector::new(); negative_end.insert(doc.peer_id(), -1); - let json = doc.export_json_updates(&VersionVector::default(), &negative_end); + let json = doc + .export_json_updates(&VersionVector::default(), &negative_end) + .unwrap(); let restored = LoroDoc::new(); restored.import_json_updates(json)?; assert_eq!(restored.get_text("text").to_string(), ""); @@ -492,7 +500,8 @@ fn import_json_updates_rejects_negative_op_counters() -> anyhow::Result<()> { doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); json.changes[0].id.counter = -1; json.changes[0].ops[0].counter = -1; @@ -513,7 +522,8 @@ fn import_json_updates_rejects_negative_dependency_counters() -> anyhow::Result< doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); json.changes[0].deps.push(ID::new(77, -1)); let err = LoroDoc::new().import_json_updates(json).unwrap_err(); @@ -576,6 +586,7 @@ fn json_update_schema_covers_list_map_text_tree_and_movable_list_ops() -> anyhow serde_json::to_value(&changes)?, serde_json::to_value( &doc.export_json_updates_without_peer_compression(&VersionVector::default(), &end) + .unwrap() .changes )? ); @@ -671,7 +682,9 @@ fn json_update_schema_covers_list_map_text_tree_and_movable_list_ops() -> anyhow assert!(saw_tree_move); assert!(saw_tree_delete); - let compressed = doc.export_json_updates(&VersionVector::default(), &end); + let compressed = doc + .export_json_updates(&VersionVector::default(), &end) + .unwrap(); assert!(compressed.peers.is_some()); assert_eq!(compressed.changes.len(), changes.len()); @@ -700,7 +713,8 @@ fn import_json_updates_rejects_non_contiguous_op_counters() -> anyhow::Result<() doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); json.changes[0].ops[0].counter += 1; let err = LoroDoc::new().import_json_updates(json).unwrap_err(); @@ -721,7 +735,8 @@ fn import_json_updates_rejects_mismatched_created_container_id() -> anyhow::Resu doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); for op in &mut json.changes[0].ops { if let JsonOpContent::Map(JsonMapOp::Insert { value: LoroValue::Container(id), @@ -751,10 +766,12 @@ fn import_json_updates_rejects_mismatched_list_created_container_ids() -> anyhow .insert_container(0, LoroMap::new())?; list_doc.commit(); - let mut list_json = list_doc.export_json_updates_without_peer_compression( - &VersionVector::default(), - &list_doc.oplog_vv(), - ); + let mut list_json = list_doc + .export_json_updates_without_peer_compression( + &VersionVector::default(), + &list_doc.oplog_vv(), + ) + .unwrap(); for op in &mut list_json.changes[0].ops { if let JsonOpContent::List(JsonListOp::Insert { value, .. }) = &mut op.content { if let Some(LoroValue::Container(id)) = value.first_mut() { @@ -777,10 +794,12 @@ fn import_json_updates_rejects_mismatched_list_created_container_ids() -> anyhow movable.set_container(0, LoroText::new())?; movable_doc.commit(); - let mut movable_json = movable_doc.export_json_updates_without_peer_compression( - &VersionVector::default(), - &movable_doc.oplog_vv(), - ); + let mut movable_json = movable_doc + .export_json_updates_without_peer_compression( + &VersionVector::default(), + &movable_doc.oplog_vv(), + ) + .unwrap(); for op in &mut movable_json.changes[0].ops { if let JsonOpContent::MovableList(JsonMovableListOp::Set { value: LoroValue::Container(id), @@ -811,7 +830,8 @@ fn import_json_updates_rejects_tree_create_target_not_matching_op_id() -> anyhow doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); for op in &mut json.changes[0].ops { if let JsonOpContent::Tree(JsonTreeOp::Create { target, .. }) = &mut op.content { *target = TreeID { @@ -849,7 +869,8 @@ fn import_json_updates_rejects_nested_container_values() -> anyhow::Result<()> { doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); for op in &mut json.changes[0].ops { if let JsonOpContent::Map(JsonMapOp::Insert { value, .. }) = &mut op.content { *value = nested_container_value(76, op.counter); @@ -869,7 +890,8 @@ fn import_json_updates_rejects_nested_container_values() -> anyhow::Result<()> { doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); for op in &mut json.changes[0].ops { if let JsonOpContent::List(JsonListOp::Insert { value, .. }) = &mut op.content { value[0] = nested_container_value(77, op.counter); @@ -891,7 +913,8 @@ fn import_json_updates_rejects_nested_container_values() -> anyhow::Result<()> { doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); for op in &mut json.changes[0].ops { if let JsonOpContent::Text(JsonTextOp::Mark { style_value, .. }) = &mut op.content { *style_value = nested_container_value(78, op.counter); @@ -913,7 +936,8 @@ fn import_json_updates_rejects_nested_container_values() -> anyhow::Result<()> { doc.commit(); let mut json = doc - .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()); + .export_json_updates_without_peer_compression(&VersionVector::default(), &doc.oplog_vv()) + .unwrap(); for op in &mut json.changes[0].ops { if let JsonOpContent::MovableList(JsonMovableListOp::Set { value, .. }) = &mut op.content { *value = nested_container_value(79, op.counter); diff --git a/crates/loro/tests/failed_import_state_rollback.rs b/crates/loro/tests/failed_import_state_rollback.rs index 4b841499c..49cc7eb9d 100644 --- a/crates/loro/tests/failed_import_state_rollback.rs +++ b/crates/loro/tests/failed_import_state_rollback.rs @@ -54,7 +54,8 @@ fn snapshot_bad_and_good_update() -> (Vec, Vec, Vec) { e.get_list("list").insert(0, 1).unwrap(); e.get_list("list").insert(1, 2).unwrap(); e.commit(); - let mut json = serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv()).unwrap()).unwrap(); let last = json["changes"].as_array_mut().unwrap().last_mut().unwrap()["ops"] .as_array_mut() .unwrap() @@ -120,7 +121,8 @@ fn failed_import_leaves_state_at_freed_indices() { e.get_list("list").insert(0, 1).unwrap(); e.get_list("list").insert(1, 2).unwrap(); e.commit(); - let mut json = serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv()).unwrap()).unwrap(); let last = json["changes"].as_array_mut().unwrap().last_mut().unwrap()["ops"] .as_array_mut() .unwrap() @@ -294,7 +296,8 @@ fn handlers_created_during_a_failed_import_keep_their_containers() { e.get_list("list").insert(0, 1).unwrap(); e.get_list("list").insert(1, 2).unwrap(); e.commit(); - let mut json = serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv()).unwrap()).unwrap(); let last = json["changes"].as_array_mut().unwrap().last_mut().unwrap()["ops"] .as_array_mut() .unwrap() diff --git a/crates/loro/tests/import_reused_peer_id.rs b/crates/loro/tests/import_reused_peer_id.rs index bc885d459..7829e04e4 100644 --- a/crates/loro/tests/import_reused_peer_id.rs +++ b/crates/loro/tests/import_reused_peer_id.rs @@ -97,7 +97,9 @@ fn import_overlapping_change_with_different_content_is_rejected() -> LoroResult< assert!(!history.is_detached()); } - let json = other.export_json_updates(&Default::default(), &other.oplog_vv()); + let json = other + .export_json_updates(&Default::default(), &other.oplog_vv()) + .unwrap(); let err = history.import_json_updates(json).unwrap_err(); assert_used_op_id(err, ID::new(7, 0)); assert_unchanged(&history, &before); @@ -261,7 +263,9 @@ fn reimporting_known_history_with_new_changes_still_succeeds() -> LoroResult<()> let full_updates = source.export(ExportMode::all_updates())?; let snapshot = source.export(ExportMode::Snapshot)?; - let json = source.export_json_updates(&Default::default(), &source.oplog_vv()); + let json = source + .export_json_updates(&Default::default(), &source.oplog_vv()) + .unwrap(); let json_updates = serde_json::to_string(&json).unwrap(); for (name, import) in [ @@ -361,7 +365,9 @@ fn rejected_import_inside_catch_unwind_can_drop_the_doc() { /// JSON updates for `doc`'s last change, moved to `peer` at `counter` with `deps`. fn forged_json_change(doc: &LoroDoc, peer: u64, counter: i32, deps: Vec) -> loro::JsonSchema { - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); json.changes.drain(..json.changes.len() - 1); let change = json.changes.last_mut().unwrap(); let shift = counter - change.id.counter; @@ -411,7 +417,9 @@ fn import_change_that_skips_counters_is_rejected() -> LoroResult<()> { helper.set_peer_id(9)?; helper.get_map("m").insert("k", 1)?; helper.commit(); - let mut json = helper.export_json_updates(&Default::default(), &helper.oplog_vv()); + let mut json = helper + .export_json_updates(&Default::default(), &helper.oplog_vv()) + .unwrap(); let forged = forged_json_change(&writer, 7, 5, vec![ID::new(1, 0)]); // Peer index 1 is peer 7, index 0 is peer 9. json.peers = Some(vec![9, 7]); diff --git a/crates/loro/tests/integration_test/detached_editing_test.rs b/crates/loro/tests/integration_test/detached_editing_test.rs index 65ecb96c5..1a0952c11 100644 --- a/crates/loro/tests/integration_test/detached_editing_test.rs +++ b/crates/loro/tests/integration_test/detached_editing_test.rs @@ -64,13 +64,13 @@ fn allow_editing_on_detached_mode_when_detached_editing_is_enabled() { assert_eq!(doc.get_text("text").to_string(), "Hello alice!"); assert_eq!(&**string.lock().unwrap(), "Hello alice!"); assert_ne!(doc.state_frontiers(), doc.oplog_frontiers()); - assert_ne!(doc.oplog_vv(), doc.state_vv()); + assert_ne!(doc.oplog_vv(), doc.state_vv().unwrap()); assert_eq!(doc.state_frontiers(), Frontiers::from(ID::new(2, 6))); assert_eq!( doc.oplog_frontiers(), Frontiers::from([ID::new(1, 11), ID::new(2, 6)]) ); - assert_eq!(doc.state_vv(), vv!(1 => 5, 2 => 7)); + assert_eq!(doc.state_vv().unwrap(), vv!(1 => 5, 2 => 7)); assert_eq!(doc.oplog_vv(), vv!(1 => 12, 2 => 7)); doc.checkout_to_latest(); @@ -82,7 +82,7 @@ fn allow_editing_on_detached_mode_when_detached_editing_is_enabled() { assert_eq!(doc.get_text("text").to_string(), "Hello world! alice!"); assert_eq!(&**string.lock().unwrap(), "Hello world! alice!"); assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); - assert_eq!(doc.oplog_vv(), doc.state_vv()); + assert_eq!(doc.oplog_vv(), doc.state_vv().unwrap()); // New op on peer id 1 doc.set_peer_id(1).unwrap(); @@ -91,9 +91,11 @@ fn allow_editing_on_detached_mode_when_detached_editing_is_enabled() { assert_eq!(&**string.lock().unwrap(), "Hi Hello world! alice!"); assert_eq!(doc.get_text("text").to_string(), "Hi Hello world! alice!"); assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); - assert_eq!(doc.oplog_vv(), doc.state_vv()); + assert_eq!(doc.oplog_vv(), doc.state_vv().unwrap()); assert_eq!(doc.state_frontiers(), Frontiers::from_id(ID::new(1, 14))); - let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let actual = format!("{json:#?}"); let expected = r#"JsonSchema { schema_version: 1, @@ -220,13 +222,13 @@ fn allow_editing_on_detached_mode_when_detached_editing_is_enabled_2() { assert_eq!(doc.get_text("text").to_string(), "Hello alice!"); assert_eq!(&**string.lock().unwrap(), "Hello alice!"); assert_ne!(doc.state_frontiers(), doc.oplog_frontiers()); - assert_ne!(doc.oplog_vv(), doc.state_vv()); + assert_ne!(doc.oplog_vv(), doc.state_vv().unwrap()); assert_eq!(doc.state_frontiers(), Frontiers::from(ID::new(0, 6))); assert_eq!( doc.oplog_frontiers(), Frontiers::from([ID::new(1, 11), ID::new(0, 6)]) ); - assert_eq!(doc.state_vv(), vv!(1 => 5, 0 => 7)); + assert_eq!(doc.state_vv().unwrap(), vv!(1 => 5, 0 => 7)); assert_eq!(doc.oplog_vv(), vv!(1 => 12, 0 => 7)); doc.checkout_to_latest(); @@ -238,7 +240,7 @@ fn allow_editing_on_detached_mode_when_detached_editing_is_enabled_2() { assert_eq!(doc.get_text("text").to_string(), "Hello alice! world!"); assert_eq!(&**string.lock().unwrap(), "Hello alice! world!"); assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); - assert_eq!(doc.oplog_vv(), doc.state_vv()); + assert_eq!(doc.oplog_vv(), doc.state_vv().unwrap()); } #[test] diff --git a/crates/loro/tests/integration_test/redact_test.rs b/crates/loro/tests/integration_test/redact_test.rs index 1d5601576..a4b178c58 100644 --- a/crates/loro/tests/integration_test/redact_test.rs +++ b/crates/loro/tests/integration_test/redact_test.rs @@ -12,7 +12,9 @@ fn redact_text_doc() { text.insert(0, "Hello, world! This is a secret message.") .unwrap(); - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let mut range = VersionRange::new(); range.insert(1, 24, 30); redact(&mut json, range).unwrap(); @@ -33,7 +35,9 @@ fn redact_rejects_overflowing_json_counters_without_panicking() { let text = doc.get_text("text"); text.insert(0, "secret").unwrap(); - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let change = &mut json.changes[0]; change.id.counter = i32::MAX; change.ops[0].counter = i32::MAX; @@ -64,7 +68,9 @@ fn redact_map_list_insertions() { list.insert(0, "secret info").unwrap(); list.insert(1, true).unwrap(); - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let mut range = VersionRange::new(); range.insert(1, 0, 5); // Redact all operations redact(&mut json, range).unwrap(); @@ -155,7 +161,9 @@ fn redact_movable_list() { list.insert(1, "sensitive data 2").unwrap(); list.set(0, "updated sensitive data").unwrap(); - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let mut range = VersionRange::new(); range.insert(1, 0, 3); redact(&mut json, range).unwrap(); @@ -245,7 +253,9 @@ fn redact_should_keep_parent_child_relationship() { let _m = sub_map .insert_container("ll", LoroMovableList::new()) .unwrap(); - let mut json = doc.export_json_updates(&Default::default(), &doc.oplog_vv()); + let mut json = doc + .export_json_updates(&Default::default(), &doc.oplog_vv()) + .unwrap(); let mut range = VersionRange::new(); range.insert(1, 0, 100); redact(&mut json, range).unwrap(); diff --git a/crates/loro/tests/integration_test/shallow_snapshot_test.rs b/crates/loro/tests/integration_test/shallow_snapshot_test.rs index 8e0056dba..c3334aa74 100644 --- a/crates/loro/tests/integration_test/shallow_snapshot_test.rs +++ b/crates/loro/tests/integration_test/shallow_snapshot_test.rs @@ -341,8 +341,8 @@ fn the_vv_on_shallow_doc() -> anyhow::Result<()> { let new_doc = LoroDoc::new(); new_doc.import(&snapshot.unwrap()).unwrap(); assert!(!new_doc.shallow_since_vv().is_empty()); - assert_eq!(new_doc.oplog_vv(), new_doc.state_vv()); - assert_eq!(new_doc.oplog_vv(), doc.state_vv()); + assert_eq!(new_doc.oplog_vv(), new_doc.state_vv().unwrap()); + assert_eq!(new_doc.oplog_vv(), doc.state_vv().unwrap()); assert_eq!(new_doc.oplog_frontiers(), doc.oplog_frontiers()); assert_eq!(new_doc.oplog_frontiers(), new_doc.state_frontiers()); assert_eq!(new_doc.get_deep_value(), doc.get_deep_value()); @@ -351,8 +351,8 @@ fn the_vv_on_shallow_doc() -> anyhow::Result<()> { doc.commit(); let bytes = doc.export(ExportMode::all_updates()); new_doc.import(&bytes.unwrap()).unwrap(); - assert_eq!(new_doc.oplog_vv(), new_doc.state_vv()); - assert_eq!(new_doc.oplog_vv(), doc.state_vv()); + assert_eq!(new_doc.oplog_vv(), new_doc.state_vv().unwrap()); + assert_eq!(new_doc.oplog_vv(), doc.state_vv().unwrap()); assert_eq!(new_doc.oplog_frontiers(), doc.oplog_frontiers()); assert_eq!(new_doc.oplog_frontiers(), new_doc.state_frontiers()); assert_eq!(new_doc.get_deep_value(), doc.get_deep_value()); @@ -523,7 +523,8 @@ fn shallow_doc_accepts_cross_peer_op_whose_deps_include_boundary() -> anyhow::Re .as_single() .expect("incoming change must have one id"); let mut update = incoming - .export_json_updates_without_peer_compression(&before_incoming, &incoming.oplog_vv()); + .export_json_updates_without_peer_compression(&before_incoming, &incoming.oplog_vv()) + .unwrap(); assert_eq!(update.changes.len(), 1); let actual_deps = &mut update.changes[0].deps; assert_eq!(actual_deps.as_slice(), &[retained]); diff --git a/crates/loro/tests/legacy_text_delete_start_id.rs b/crates/loro/tests/legacy_text_delete_start_id.rs index 5f5114cb8..c27868840 100644 --- a/crates/loro/tests/legacy_text_delete_start_id.rs +++ b/crates/loro/tests/legacy_text_delete_start_id.rs @@ -22,7 +22,9 @@ fn legacy_history() -> LoroDoc { text.insert(1, "!").unwrap(); writer.commit(); - let mut json = writer.export_json_updates(&Default::default(), &writer.oplog_vv()); + let mut json = writer + .export_json_updates(&Default::default(), &writer.oplog_vv()) + .unwrap(); for change in json.changes.iter_mut() { let deletes: Vec = change .ops diff --git a/crates/loro/tests/loro_js_interop.rs b/crates/loro/tests/loro_js_interop.rs index 371b90fee..73c16cf7f 100644 --- a/crates/loro/tests/loro_js_interop.rs +++ b/crates/loro/tests/loro_js_interop.rs @@ -400,12 +400,18 @@ fn imports_typescript_movable_list_snapshot_with_its_metadata() { doc.import(TS_MOVABLE_SNAPSHOT) .expect("Rust should import a MovableList snapshot produced by loro.js"); assert_eq!(doc.get_deep_value(), expected.get_deep_value()); - assert_eq!(movable_list_metadata(&doc), movable_list_metadata(&expected)); + assert_eq!( + movable_list_metadata(&doc), + movable_list_metadata(&expected) + ); for update in [RUST_MOVABLE_PEER4, RUST_MOVABLE_PEER3] { doc.import(update).unwrap(); expected.import(update).unwrap(); } assert_eq!(doc.get_deep_value(), expected.get_deep_value()); - assert_eq!(movable_list_metadata(&doc), movable_list_metadata(&expected)); + assert_eq!( + movable_list_metadata(&doc), + movable_list_metadata(&expected) + ); } diff --git a/crates/loro/tests/loro_rust_test.rs b/crates/loro/tests/loro_rust_test.rs index 42df863df..7be1ec9e8 100644 --- a/crates/loro/tests/loro_rust_test.rs +++ b/crates/loro/tests/loro_rust_test.rs @@ -1278,7 +1278,7 @@ fn sync_two_shallow_docs() { assert_eq!(doc_a.get_deep_value(), doc_b.get_deep_value()); assert_eq!(doc_a.oplog_vv(), doc_b.oplog_vv()); assert_eq!(doc_a.oplog_frontiers(), doc_b.oplog_frontiers()); - assert_eq!(doc_a.state_vv(), doc_b.state_vv()); + assert_eq!(doc_a.state_vv().unwrap(), doc_b.state_vv().unwrap()); assert_eq!(doc_a.shallow_since_vv(), doc_b.shallow_since_vv()); } @@ -2092,7 +2092,7 @@ fn issue_490() -> anyhow::Result<()> { fn test_loro_doc() { let doc = LoroDoc::new(); doc.get_text("text").insert(0, "Hello").unwrap(); - doc.state_vv(); + doc.state_vv().unwrap(); } #[test] @@ -3308,7 +3308,9 @@ fn test_export_json_updates_in_shallow_snapshot() { .unwrap(); let new_doc = LoroDoc::new(); new_doc.import(&snapshot).unwrap(); - new_doc.export_json_updates(&Default::default(), &new_doc.oplog_vv()); + new_doc + .export_json_updates(&Default::default(), &new_doc.oplog_vv()) + .unwrap(); } #[test] diff --git a/crates/loro/tests/moon_transcode.rs b/crates/loro/tests/moon_transcode.rs index c7d0f50c6..b61ba4015 100644 --- a/crates/loro/tests/moon_transcode.rs +++ b/crates/loro/tests/moon_transcode.rs @@ -91,7 +91,11 @@ fn run_transcode(node_bin: &str, cli_js: &Path, input: &[u8]) -> anyhow::Result< .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - let tmp = std::env::temp_dir().join(format!("loro-moon-transcode-{}-{ts}-{}", std::process::id(), next_tmp_id())); + let tmp = std::env::temp_dir().join(format!( + "loro-moon-transcode-{}-{ts}-{}", + std::process::id(), + next_tmp_id() + )); std::fs::create_dir_all(&tmp)?; let in_path = tmp.join("in.blob"); let out_path = tmp.join("out.blob"); @@ -773,7 +777,7 @@ fn assert_updates_jsonschema_matches_rust(doc: &LoroDoc, ctx: &MoonCtx) -> anyho let moon_json = run_export_jsonschema(&ctx.node_bin, &ctx.cli_js, &updates_blob)?; let moon_value: serde_json::Value = serde_json::from_str(&moon_json)?; - let rust_schema = doc.export_json_updates(&start, &end); + let rust_schema = doc.export_json_updates(&start, &end).unwrap(); let rust_value = serde_json::to_value(&rust_schema)?; anyhow::ensure!( @@ -1221,7 +1225,7 @@ fn moon_edge_varints_and_lengths() -> anyhow::Result<()> { // Roundtrip: Moon encode-jsonschema output must be importable by Rust (large peer/key tables). let start = VersionVector::default(); let end = doc.oplog_vv(); - let schema = doc.export_json_updates(&start, &end); + let schema = doc.export_json_updates(&start, &end).unwrap(); let json = serde_json::to_string(&schema)?; let out_blob = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json)?; let doc2 = LoroDoc::new(); @@ -1371,7 +1375,7 @@ fn moon_encode_jsonschema_text_insert() -> anyhow::Result<()> { let start = VersionVector::default(); let end = doc.oplog_vv(); - let schema = doc.export_json_updates(&start, &end); + let schema = doc.export_json_updates(&start, &end).unwrap(); let json = serde_json::to_string(&schema)?; let out_blob = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json)?; @@ -1407,7 +1411,9 @@ fn moon_encode_jsonschema_cross_peer_container_refs() -> anyhow::Result<()> { let end = doc.oplog_vv(); // Full range should import on a fresh doc. - let schema0 = doc.export_json_updates(&VersionVector::default(), &end); + let schema0 = doc + .export_json_updates(&VersionVector::default(), &end) + .unwrap(); let json0 = serde_json::to_string(&schema0)?; let blob0 = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json0)?; let doc0 = LoroDoc::new(); @@ -1416,7 +1422,7 @@ fn moon_encode_jsonschema_cross_peer_container_refs() -> anyhow::Result<()> { // Incremental range should apply on top of SnapshotAt(v1). let vv_v1: VersionVector = doc.frontiers_to_vv(&frontiers_v1).unwrap(); - let schema = doc.export_json_updates(&vv_v1, &end); + let schema = doc.export_json_updates(&vv_v1, &end).unwrap(); let json = serde_json::to_string(&schema)?; let blob = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json)?; let base_snapshot = doc.export(ExportMode::SnapshotAt { @@ -1455,7 +1461,9 @@ fn moon_encode_jsonschema_random_roundtrip() -> anyhow::Result<()> { let end = doc.oplog_vv(); // Full range (empty start) should roundtrip on a fresh doc. - let schema0 = doc.export_json_updates(&VersionVector::default(), &end); + let schema0 = doc + .export_json_updates(&VersionVector::default(), &end) + .unwrap(); let json0 = serde_json::to_string(&schema0)?; let blob0 = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json0)?; let doc0 = LoroDoc::new(); @@ -1469,7 +1477,7 @@ fn moon_encode_jsonschema_random_roundtrip() -> anyhow::Result<()> { // Incremental range should apply cleanly on top of SnapshotAt(v1). let vv_v1: VersionVector = doc.frontiers_to_vv(&frontiers_v1).unwrap(); - let schema = doc.export_json_updates(&vv_v1, &end); + let schema = doc.export_json_updates(&vv_v1, &end).unwrap(); let json = serde_json::to_string(&schema)?; let blob = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json)?; @@ -1539,7 +1547,9 @@ fn moon_encode_jsonschema_richtext_roundtrip() -> anyhow::Result<()> { let end = doc.oplog_vv(); // Full range. - let schema0 = doc.export_json_updates(&VersionVector::default(), &end); + let schema0 = doc + .export_json_updates(&VersionVector::default(), &end) + .unwrap(); let json0 = serde_json::to_string(&schema0)?; let blob0 = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json0)?; let doc0 = LoroDoc::new(); @@ -1556,7 +1566,7 @@ fn moon_encode_jsonschema_richtext_roundtrip() -> anyhow::Result<()> { // Incremental range (SnapshotAt(v1) + Updates(from v1)). let vv_v1: VersionVector = doc.frontiers_to_vv(&frontiers_v1).unwrap(); - let schema = doc.export_json_updates(&vv_v1, &end); + let schema = doc.export_json_updates(&vv_v1, &end).unwrap(); let json = serde_json::to_string(&schema)?; let blob = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json)?; let base_snapshot = doc.export(ExportMode::SnapshotAt { @@ -1924,7 +1934,7 @@ fn moon_encode_jsonschema_counter() -> anyhow::Result<()> { let start = VersionVector::default(); let end = doc.oplog_vv(); - let schema = doc.export_json_updates(&start, &end); + let schema = doc.export_json_updates(&start, &end).unwrap(); let json = serde_json::to_string(&schema)?; let out_blob = run_encode_jsonschema(&ctx.node_bin, &ctx.cli_js, &json)?; diff --git a/crates/loro/tests/multi_thread_test.rs b/crates/loro/tests/multi_thread_test.rs index 75d8a5661..7c07b6205 100644 --- a/crates/loro/tests/multi_thread_test.rs +++ b/crates/loro/tests/multi_thread_test.rs @@ -421,9 +421,12 @@ mod loom_test { let reference = LoroDoc::new(); reference.import(&snapshot).unwrap(); let vv = reference.oplog_vv(); - let history = - serde_json::to_string(&reference.export_json_updates(&Default::default(), &vv)) - .unwrap(); + let history = serde_json::to_string( + &reference + .export_json_updates(&Default::default(), &vv) + .unwrap(), + ) + .unwrap(); // Another peer creates a container, so decoding its change registers one. The // empty change after it fails the decode. let e = LoroDoc::new(); @@ -435,7 +438,8 @@ mod loom_test { .insert("q", 1) .unwrap(); e.commit(); - let mut json = serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv()).unwrap()).unwrap(); let changes = json["changes"].as_array_mut().unwrap(); let mut empty = changes.last().unwrap().clone(); empty["ops"] = serde_json::json!([]); @@ -466,9 +470,10 @@ mod loom_test { h0.join().unwrap(); h1.join().unwrap(); assert!(doc.has_container(&f.meta)); - let history = - serde_json::to_string(&doc.export_json_updates(&Default::default(), &f.vv)) - .unwrap(); + let history = serde_json::to_string( + &doc.export_json_updates(&Default::default(), &f.vv).unwrap(), + ) + .unwrap(); assert_eq!(history, f.history); }); } diff --git a/crates/loro/tests/perf_history_lazy_load.rs b/crates/loro/tests/perf_history_lazy_load.rs new file mode 100644 index 000000000..8f58c83f9 --- /dev/null +++ b/crates/loro/tests/perf_history_lazy_load.rs @@ -0,0 +1,114 @@ +use loro::{ExportMode, Frontiers, LoroDoc, ID}; +use std::{hint::black_box, time::Instant}; + +/// Cold snapshot import, lazy historical checkout, and concurrent update imports. +/// Run in release, with one test thread, on both revisions; compare three repeats. +#[test] +#[ignore] +fn perf_history_lazy_load() { + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + source.set_change_merge_interval(0); + let text = source.get_text("text"); + for _ in 0..600 { + text.insert(0, &"x".repeat(30)).unwrap(); + source.commit(); + } + source.set_peer_id(2).unwrap(); + for _ in 0..300 { + text.insert(0, &"y".repeat(30)).unwrap(); + source.commit(); + } + source.get_map("map").insert("key", 1).unwrap(); + source.commit(); + let snapshot = source.export(ExportMode::Snapshot).unwrap(); + let remote = LoroDoc::new(); + remote.set_peer_id(3).unwrap(); + remote.get_text("text").insert(0, "concurrent").unwrap(); + let updates = remote.export(ExportMode::all_updates()).unwrap(); + let map_remote = LoroDoc::new(); + map_remote.set_peer_id(4).unwrap(); + map_remote.get_map("map").insert("key", 2).unwrap(); + let map_updates = map_remote.export(ExportMode::all_updates()).unwrap(); + for repeat in 1..=3 { + let mut import = std::time::Duration::ZERO; + let mut checkout = import; + let mut concurrent = import; + let mut map_import = import; + for _ in 0..30 { + let doc = LoroDoc::new(); + let start = Instant::now(); + doc.import(black_box(&snapshot)).unwrap(); + import += start.elapsed(); + let start = Instant::now(); + doc.checkout(&Frontiers::from(ID::new(1, 10))).unwrap(); + checkout += start.elapsed(); + black_box(doc.get_deep_value()); + let doc = LoroDoc::new(); + doc.import(&snapshot).unwrap(); + let start = Instant::now(); + doc.import(black_box(&updates)).unwrap(); + concurrent += start.elapsed(); + black_box(doc.get_deep_value()); + let doc = LoroDoc::new(); + doc.import(&snapshot).unwrap(); + let start = Instant::now(); + doc.import(black_box(&map_updates)).unwrap(); + map_import += start.elapsed(); + black_box(doc.get_deep_value()); + } + println!( + "repeat={repeat} snapshot_import_us={} cold_checkout_us={} concurrent_import_us={} map_import_us={}", + import.as_micros(), + checkout.as_micros(), + concurrent.as_micros(), + map_import.as_micros() + ); + } +} + +/// A one-op, causal map import with a large version vector. Header warming drains +/// the DAG's unparsed_vv without parsing old block bodies. Setup is not timed. +#[test] +#[ignore] +fn perf_map_import_many_peers() { + for peers in [1_000u64, 10_000] { + let source = LoroDoc::new(); + let map = source.get_map("map"); + for peer in 1..=peers { + source.set_peer_id(peer).unwrap(); + map.insert("key", peer as i64).unwrap(); + source.commit(); + } + let old_vv = source.oplog_vv(); + let snapshot = source.export(ExportMode::Snapshot).unwrap(); + source.set_peer_id(peers + 1).unwrap(); + map.insert("key", -1).unwrap(); + let update = source.export(ExportMode::updates(&old_vv)).unwrap(); + for warm_headers in [false, true] { + for repeat in 1..=3 { + let mut elapsed = std::time::Duration::ZERO; + for _ in 0..30 { + let doc = LoroDoc::new(); + doc.import(&snapshot).unwrap(); + if warm_headers { + doc.with_oplog(|oplog| { + for peer in 1..=peers { + black_box(oplog.dag().get_lamport(&ID::new(peer, 0))); + } + }); + } + let start = Instant::now(); + doc.import(black_box(&update)).unwrap(); + elapsed += start.elapsed(); + assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); + black_box(doc.get_deep_value()); + } + println!( + "peers={peers} warm_headers={warm_headers} repeat={repeat} map_import_us={}", + elapsed.as_micros() + ); + } + } + } +} diff --git a/crates/loro/tests/perf_unknown_container_apply_diff.rs b/crates/loro/tests/perf_unknown_container_apply_diff.rs index a9e754001..29823f647 100644 --- a/crates/loro/tests/perf_unknown_container_apply_diff.rs +++ b/crates/loro/tests/perf_unknown_container_apply_diff.rs @@ -40,7 +40,8 @@ fn doc_with_unknown_element() -> LoroDoc { src.commit(); let cid = loro::ContainerTrait::id(&text).to_string(); let json = serde_json::to_string( - &src.export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()), + &src.export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()) + .unwrap(), ) .unwrap() .replace(&cid, &cid.replace(":Text", ":Unknown(9)")); @@ -214,8 +215,9 @@ fn doc_with_mergeable_list(unknown: bool) -> LoroDoc { list.insert_container(N / 2, LoroText::new()).unwrap(); } src.commit(); - let mut json = - src.export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()); + let mut json = src + .export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()) + .unwrap(); for change in json.changes.iter_mut() { for op in change.ops.iter_mut() { if let loro::JsonOpContent::List(loro::JsonListOp::Insert { value, .. }) = @@ -261,8 +263,9 @@ fn perf_full_state_revival() { /// Replays `src` with the ids of its Text containers (which must have no ops) /// turned into `Unknown(9)`, editing the typed JSON so mergeable markers stay. fn forge_texts(src: &LoroDoc) -> LoroDoc { - let mut json = - src.export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()); + let mut json = src + .export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()) + .unwrap(); let forge = |v: &mut LoroValue| { if let LoroValue::Container(ContainerID::Normal { peer, diff --git a/crates/loro/tests/root_visibility_replay.rs b/crates/loro/tests/root_visibility_replay.rs index 6bc9e8414..333337c11 100644 --- a/crates/loro/tests/root_visibility_replay.rs +++ b/crates/loro/tests/root_visibility_replay.rs @@ -67,7 +67,10 @@ fn every_import_path_agrees() { let from_json = LoroDoc::new(); from_json - .import_json_updates(a.export_json_updates(&Default::default(), &a.oplog_vv())) + .import_json_updates( + a.export_json_updates(&Default::default(), &a.oplog_vv()) + .unwrap(), + ) .unwrap(); assert_eq!(value(&from_json), expected, "json"); diff --git a/crates/loro/tests/shallow_checkout_equal_value.rs b/crates/loro/tests/shallow_checkout_equal_value.rs index c1109a243..cfbd5295b 100644 --- a/crates/loro/tests/shallow_checkout_equal_value.rs +++ b/crates/loro/tests/shallow_checkout_equal_value.rs @@ -341,7 +341,9 @@ fn equal_value_winner_change_emits_no_events() { })); let changed_only = |events: &[String]| { assert!(events.iter().all(|e| !e.contains("same")), "{events:?}"); - assert!(events.iter().any(|e| e.contains("new") || e.contains("old"))); + assert!(events + .iter() + .any(|e| e.contains("new") || e.contains("old"))); }; doc.checkout(&a).unwrap(); doc.checkout(&c).unwrap(); @@ -366,7 +368,12 @@ fn equal_value_winner_change_emits_no_events() { changed_only(&received.lock().unwrap()); assert_eq!(receiver.get_map("map").get_last_editor("x"), Some(1)); assert_eq!( - receiver.get_map("map").get("x").unwrap().into_value().unwrap(), + receiver + .get_map("map") + .get("x") + .unwrap() + .into_value() + .unwrap(), LoroValue::from("same") ); } diff --git a/crates/loro/tests/unknown_container_apply_diff.rs b/crates/loro/tests/unknown_container_apply_diff.rs index c44bce542..c37d01c36 100644 --- a/crates/loro/tests/unknown_container_apply_diff.rs +++ b/crates/loro/tests/unknown_container_apply_diff.rs @@ -47,7 +47,8 @@ fn doc_with_unknown_in(setup: impl Fn(&LoroDoc) -> ContainerID) -> (LoroDoc, Con let text_id = setup(&doc); doc.commit(); let json = serde_json::to_string( - &doc.export_json_updates_without_peer_compression(&Default::default(), &doc.oplog_vv()), + &doc.export_json_updates_without_peer_compression(&Default::default(), &doc.oplog_vv()) + .unwrap(), ) .unwrap(); let text_cid = text_id.to_string(); @@ -363,7 +364,8 @@ fn edited_unknown_child_updates() -> String { counter.increment(2.0).unwrap(); doc.commit(); let json = serde_json::to_string( - &doc.export_json_updates_without_peer_compression(&Default::default(), &doc.oplog_vv()), + &doc.export_json_updates_without_peer_compression(&Default::default(), &doc.oplog_vv()) + .unwrap(), ) .unwrap(); assert!(json.contains("cid:0@1:Counter") && json.contains(r#""type":"counter""#)); diff --git a/crates/loro/tests/unknown_container_support/mod.rs b/crates/loro/tests/unknown_container_support/mod.rs index 9657606bc..5c44edc46 100644 --- a/crates/loro/tests/unknown_container_support/mod.rs +++ b/crates/loro/tests/unknown_container_support/mod.rs @@ -43,8 +43,9 @@ pub fn forge_as(build: impl FnOnce(&LoroDoc), unknown: bool) -> LoroDoc { src.set_peer_id(1).unwrap(); build(&src); src.commit(); - let mut json = - src.export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()); + let mut json = src + .export_json_updates_without_peer_compression(&Default::default(), &src.oplog_vv()) + .unwrap(); for change in json.changes.iter_mut().filter(|_| unknown) { for op in change.ops.iter_mut() { forge_id(&mut op.container); diff --git a/crates/loro/tests/unparsable_history.rs b/crates/loro/tests/unparsable_history.rs new file mode 100644 index 000000000..aaac12d90 --- /dev/null +++ b/crates/loro/tests/unparsable_history.rs @@ -0,0 +1,206 @@ +//! Unreadable lazy history must come back as `DecodeError` from the public +//! reads that used to unwrap it: version, JSON export, cursors, `apply_diff` +//! rollback, the next shallow import, and `fork_at`. + +use loro::event::{Diff, DiffBatch}; +use loro::{ + ContainerID, ContainerType, ExportMode, Frontiers, LoroDoc, LoroError, TextDelta, + VersionVector, ID, +}; + +const FIXTURE: &[u8] = + include_bytes!("../../loro-wasm/tests/fixtures/unparsable-history.snapshot.bin"); + +fn assert_decode(err: &LoroError) { + assert!( + matches!(err, LoroError::DecodeError(_)), + "expected DecodeError, got {err:?}" + ); +} + +fn snapshot_with_truncated_peer1_block(snapshot: &[u8]) -> (Vec, ID) { + use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; + use std::ops::Bound; + let mut at = 22; + let mut sections = Vec::new(); + for _ in 0..3 { + let len = u32::from_le_bytes(snapshot[at..at + 4].try_into().unwrap()) as usize; + at += 4; + sections.push(snapshot[at..at + len].to_vec()); + at += len; + } + let mut kv = MemKvStore::new(MemKvConfig::new()); + kv.import_all(sections[0].clone().into()).unwrap(); + let blocks: Vec<_> = kv + .scan(Bound::Unbounded, Bound::Unbounded) + .filter(|(key, _)| key.len() == 12 && ID::from_bytes(key).peer == 1) + .collect(); + assert!( + blocks.len() >= 3, + "need a non-frontier peer-1 block, found {}", + blocks.len() + ); + let (key, bytes) = &blocks[0]; + let id = ID::from_bytes(key).inc(1); + kv.set(key, bytes.slice(..bytes.len() / 2)); + sections[0] = kv.export_all().to_vec(); + let mut forged = snapshot[..22].to_vec(); + for section in sections { + forged.extend_from_slice(&(section.len() as u32).to_le_bytes()); + forged.extend_from_slice(§ion); + } + let checksum = xxhash_rust::xxh32::xxh32(&forged[20..], u32::from_le_bytes(*b"LORO")); + forged[16..20].copy_from_slice(&checksum.to_le_bytes()); + (forged, id) +} + +#[test] +fn state_vv_of_the_fixture_uses_the_cached_oplog_vv() { + let doc = LoroDoc::new(); + doc.import(FIXTURE).unwrap(); + assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); + let vv = doc.state_vv().unwrap(); + assert_eq!(vv, doc.oplog_vv()); + assert_eq!(doc.try_state_vv().unwrap(), doc.oplog_vv()); + + // The tip fast path does not walk this non-frontier id. The block that + // contains `1@1` is truncated, so the walk must fail closed. + let err = doc + .try_frontiers_to_vv(&Frontiers::from(ID::new(1, 1))) + .unwrap_err(); + assert_decode(&err); + let err = doc.checkout(&Frontiers::from(ID::new(1, 1))).unwrap_err(); + assert_decode(&err); + assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); +} + +#[test] +fn json_export_of_a_truncated_peer_returns_decode_error() { + let doc = LoroDoc::new(); + doc.import(FIXTURE).unwrap(); + let mut start = VersionVector::default(); + start.set_end(ID::new(1, 2)); + start.set_end(ID::new(2, 1)); + let err = doc + .try_export_json_updates(&start, &doc.oplog_vv()) + .unwrap_err(); + assert_decode(&err); + let err = doc + .export_json_updates(&start, &doc.oplog_vv()) + .unwrap_err(); + assert_decode(&err); +} + +#[test] +fn apply_diff_does_not_panic_or_advance_the_doc_when_history_is_unreadable() { + let doc = LoroDoc::new(); + doc.import(FIXTURE).unwrap(); + let frontiers = doc.state_frontiers(); + let value = doc.get_deep_value(); + let mut batch = DiffBatch::default(); + batch + .push( + ContainerID::new_root("text", ContainerType::Text), + Diff::Text(vec![TextDelta::Insert { + insert: "x".into(), + attributes: None, + }]), + ) + .unwrap(); + batch + .push( + ContainerID::new_normal(ID::new(99, 0), ContainerType::Text), + Diff::Text(vec![]), + ) + .unwrap(); + let err = doc.apply_diff(batch).unwrap_err(); + assert!(err.to_string().contains("Decode error"), "{err:?}"); + assert_eq!(doc.state_frontiers(), frontiers); + assert_eq!(doc.oplog_frontiers(), frontiers); + assert_eq!(doc.get_deep_value(), value); + doc.get_text("text").insert(0, "still works").unwrap(); +} + +#[test] +fn deleted_cursor_reports_unreadable_history() { + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + source.set_change_merge_interval(-1); + let text = source.get_text("text"); + text.insert(0, "cursor target").unwrap(); + source.commit(); + let cursor = text.get_cursor(0, Default::default()).unwrap(); + text.delete(0, text.len_unicode()).unwrap(); + source.commit(); + for i in 0..600 { + text.insert(0, &format!("line-{i:04}-xxxxxxxxxxxxxxxxxxxxxxxx")) + .unwrap(); + source.get_map("map").insert("key", i).unwrap(); + source.commit(); + } + let snapshot = source.export(ExportMode::Snapshot).unwrap(); + let (forged, _) = snapshot_with_truncated_peer1_block(&snapshot); + let doc = LoroDoc::new(); + doc.import(&forged).unwrap(); + let err = doc.get_cursor_pos(&cursor).unwrap_err(); + assert!( + matches!(err, loro::CannotFindRelativePosition::HistoryUnreadable), + "{err:?}" + ); + let err = doc.try_get_cursor_pos(&cursor).unwrap_err(); + assert_decode(&err); +} + +#[test] +fn shallow_import_of_a_later_update_returns_decode_error_without_advancing() { + let root = LoroDoc::new(); + root.set_peer_id(2).unwrap(); + root.get_map("root").insert("key", 0).unwrap(); + root.commit(); + let shallow_root = root.oplog_frontiers(); + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + source.set_change_merge_interval(-1); + source + .import(&root.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + for i in 0..600 { + source + .get_text("text") + .insert(0, &format!("line-{i:04}-xxxxxxxxxxxxxxxxxxxxxxxx")) + .unwrap(); + source.get_map("map").insert("key", i).unwrap(); + source.commit(); + } + let shallow = source + .export(ExportMode::shallow_snapshot(&shallow_root)) + .unwrap(); + let (forged, id) = snapshot_with_truncated_peer1_block(&shallow); + let remote = source.fork_at(&Frontiers::from(id)).unwrap(); + remote.set_peer_id(9).unwrap(); + remote.get_map("incoming").insert("key", 1).unwrap(); + let update = remote + .export(ExportMode::updates(&source.oplog_vv())) + .unwrap(); + + let doc = LoroDoc::new(); + doc.import(&forged).unwrap(); + assert!(doc.is_shallow()); + let frontiers = doc.state_frontiers(); + let value = doc.get_deep_value(); + assert_eq!(doc.oplog_frontiers(), frontiers); + let err = doc.import(&update).unwrap_err(); + assert_decode(&err); + assert_eq!(doc.state_frontiers(), frontiers); + assert_eq!(doc.oplog_frontiers(), frontiers); + assert_eq!(doc.get_deep_value(), value); +} + +#[test] +fn fork_at_returns_decode_error_not_unknown() { + let doc = LoroDoc::new(); + doc.import(FIXTURE).unwrap(); + let err = doc.fork_at(&Frontiers::from(ID::new(1, 1))).unwrap_err(); + assert_decode(&err); + assert_eq!(doc.oplog_frontiers(), doc.state_frontiers()); +} diff --git a/crates/loro/tests/unregistered_container_parent.rs b/crates/loro/tests/unregistered_container_parent.rs index a11c96b31..1f2e46125 100644 --- a/crates/loro/tests/unregistered_container_parent.rs +++ b/crates/loro/tests/unregistered_container_parent.rs @@ -364,7 +364,8 @@ fn failed_import_that_loads_an_old_change() { list.insert(0, "seed").unwrap(); list.insert(1, "tail").unwrap(); e.commit(); - let mut json = serde_json::to_value(e.export_json_updates(&b_vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&b_vv, &e.oplog_vv()).unwrap()).unwrap(); let last_op = json["changes"].as_array_mut().unwrap().last_mut().unwrap()["ops"] .as_array_mut() .unwrap() @@ -394,7 +395,11 @@ fn failed_import_that_loads_an_old_change() { // The history, including the change the failed import loaded, must still name the // right containers. let history = |doc: &LoroDoc| { - serde_json::to_string(&doc.export_json_updates(&Default::default(), &a.oplog_vv())).unwrap() + serde_json::to_string( + &doc.export_json_updates(&Default::default(), &a.oplog_vv()) + .unwrap(), + ) + .unwrap() }; assert_eq!(history(&dst), history(&a)); assert!(dst.get_tree("tree").get_meta(child).unwrap().is_deleted()); @@ -436,7 +441,8 @@ fn failed_import_that_loads_old_movable_list_values() { e.get_list("list").insert(0, 1).unwrap(); e.get_list("list").insert(1, 2).unwrap(); e.commit(); - let mut json = serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv()).unwrap()).unwrap(); let last_op = json["changes"].as_array_mut().unwrap().last_mut().unwrap()["ops"] .as_array_mut() .unwrap() @@ -654,7 +660,8 @@ fn queries_race_with_failing_imports() { .unwrap(); e.commit(); } - let mut json = serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv())).unwrap(); + let mut json = + serde_json::to_value(e.export_json_updates(&vv, &e.oplog_vv()).unwrap()).unwrap(); let changes = json["changes"].as_array_mut().unwrap(); let mut empty = changes.last().unwrap().clone(); empty["ops"] = serde_json::json!([]); @@ -673,7 +680,7 @@ fn queries_race_with_failing_imports() { let old_deps = o.export(ExportMode::all_updates()).unwrap(); let history = |doc: &LoroDoc, from: &VersionVector| { - serde_json::to_string(&doc.export_json_updates(from, &vv)).unwrap() + serde_json::to_string(&doc.export_json_updates(from, &vv).unwrap()).unwrap() }; let reference = import(&a.export(ExportMode::all_updates()).unwrap()); let empty_vv = VersionVector::new(); @@ -950,3 +957,346 @@ mod random { } } } + +/// Rewrite a change-block value and both checksum layers, as in the independent +/// #1172 verifier. Import must remain lazy: no earlier query records the failure. +fn snapshot_with_truncated_history(snapshot: &[u8], block_index: usize) -> (Vec, ID) { + snapshot_with_broken_history(snapshot, block_index, true) +} + +fn snapshot_with_broken_history( + snapshot: &[u8], + block_index: usize, + truncate: bool, +) -> (Vec, ID) { + use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; + use std::ops::Bound; + let mut at = 22; + let mut sections = Vec::new(); + for _ in 0..3 { + let len = u32::from_le_bytes(snapshot[at..at + 4].try_into().unwrap()) as usize; + at += 4; + sections.push(snapshot[at..at + len].to_vec()); + at += len; + } + let mut kv = MemKvStore::new(MemKvConfig::new()); + kv.import_all(sections[0].clone().into()).unwrap(); + let blocks: Vec<_> = kv + .scan(Bound::Unbounded, Bound::Unbounded) + .filter(|(key, _)| key.len() == 12 && ID::from_bytes(key).peer == 1) + .collect(); + assert!(blocks.len() >= 3); + let index = match block_index { + 0 => 0, + 1 => blocks.len() / 2, + _ => blocks.len() - 1, + }; + let (key, bytes) = &blocks[index]; + let id = ID::from_bytes(key).inc(1); + if truncate { + kv.set(key, bytes.slice(..bytes.len() / 2)); + } else { + // Keep the header intact; the compressed body fails only when parsed. + let mut body = bytes.to_vec(); + let at = body.len() - 3; + body[at] ^= 0xff; + kv.set(key, body.into()); + } + sections[0] = kv.export_all().to_vec(); + let mut forged = snapshot[..22].to_vec(); + for section in sections { + forged.extend_from_slice(&(section.len() as u32).to_le_bytes()); + forged.extend_from_slice(§ion); + } + let checksum = xxhash_rust::xxh32::xxh32(&forged[20..], u32::from_le_bytes(*b"LORO")); + forged[16..20].copy_from_slice(&checksum.to_le_bytes()); + (forged, id) +} + +#[test] +fn register_only_import_rolls_back_when_only_old_block_headers_were_read() { + use loro::Frontiers; + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + source.set_change_merge_interval(-1); + for i in 0..600 { + source + .get_text("text") + .insert(0, &format!("line-{i:04}-xxxxxxxxxxxxxxxxxxxxxxxx")) + .unwrap(); + source.get_map("map").insert("key", i).unwrap(); + source.commit(); + } + source.set_peer_id(2).unwrap(); + for _ in 0..300 { + source.get_text("text").insert(0, &"y".repeat(30)).unwrap(); + source.commit(); + } + let snapshot = source.export(ExportMode::Snapshot).unwrap(); + let remote = LoroDoc::new(); + remote.set_peer_id(10).unwrap(); + remote.get_map("map").insert("key", -1).unwrap(); + let updates = remote.export(ExportMode::all_updates()).unwrap(); + for block in 0..3 { + let (forged, _) = snapshot_with_broken_history(&snapshot, block, false); + if block == 0 { + if let Some(path) = std::env::var_os("LORO_BODY_FIXTURE") { + std::fs::write(path, &forged).unwrap(); + } + } + let doc = LoroDoc::new(); + doc.import(&forged).unwrap(); + // These load every DAG header, draining unparsed_vv, but not block bodies. + for peer in [1, 2] { + assert!(doc + .try_frontiers_to_vv(&Frontiers::from(ID::new(peer, 0))) + .unwrap() + .is_some()); + } + let value = doc.get_deep_value(); + let frontiers = doc.state_frontiers(); + let vv = doc.oplog_vv(); + assert_eq!(doc.oplog_frontiers(), frontiers); + let error = doc.import(&updates).unwrap_err(); + assert!( + error.to_string().contains("cannot parse change block"), + "block={block}: {error}" + ); + assert_eq!(doc.get_deep_value(), value, "block={block}"); + assert_eq!(doc.state_frontiers(), frontiers, "block={block}"); + assert_eq!(doc.oplog_frontiers(), frontiers, "block={block}"); + assert_eq!(doc.oplog_vv(), vv, "block={block}"); + assert!(!doc.is_detached(), "block={block}"); + assert!(doc + .checkout(&frontiers) + .unwrap_err() + .to_string() + .contains("cannot parse change block")); + } +} + +#[test] +fn fallible_history_readers_reject_a_truncated_block_on_the_first_read() { + use loro::{Frontiers, VersionVector}; + let source = LoroDoc::new(); + source.set_peer_id(1).unwrap(); + source.set_change_merge_interval(-1); + for i in 0..600 { + source + .get_text("text") + .insert(0, &format!("line-{i:04}-xxxxxxxxxxxxxxxxxxxxxxxx")) + .unwrap(); + source.get_map("map").insert("key", i).unwrap(); + source.commit(); + } + source.set_peer_id(2).unwrap(); + for _ in 0..300 { + source.get_text("text").insert(0, &"y".repeat(30)).unwrap(); + source.commit(); + } + let snapshot = source.export(ExportMode::Snapshot).unwrap(); + let remote = LoroDoc::new(); + remote.set_peer_id(9).unwrap(); + remote.get_text("text").insert(0, "concurrent").unwrap(); + let updates = remote.export(ExportMode::all_updates()).unwrap(); + // Register-only imports normally skip state-apply rollback. Lazy history can + // still fail after these changes are appended, so test that path separately. + let map_remote = LoroDoc::new(); + map_remote.set_peer_id(10).unwrap(); + map_remote.get_map("map").insert("key", -1).unwrap(); + let map_updates = map_remote.export(ExportMode::all_updates()).unwrap(); + let json = remote + .export_json_updates_without_peer_compression(&VersionVector::default(), &remote.oplog_vv()) + .unwrap(); + for block in 0..3 { + let (forged, id) = snapshot_with_truncated_history(&snapshot, block); + // Regenerate the WASM regression fixture from the same public-API rewrite: + // LORO_HISTORY_FIXTURE= cargo test -p loro --test + // unregistered_container_parent fallible_history_readers + if block == 0 { + if let Some(path) = std::env::var_os("LORO_HISTORY_FIXTURE") { + std::fs::write(path, &forged).unwrap(); + } + } + let mid = Frontiers::from(id); + let dependent = source.fork_at(&mid).unwrap(); + dependent + .get_text("text") + .insert(0, "depends on the broken block") + .unwrap(); + let dependent_updates = dependent + .export(ExportMode::updates(&source.oplog_vv())) + .unwrap(); + let latest = source.oplog_frontiers(); + // This vector/frontier forces a two-peer walk through the broken block. + let mut vv = VersionVector::default(); + vv.set_end(id.inc(1)); + vv.set_end(latest.as_single().unwrap().inc(1)); + let redundant = Frontiers::from(vec![id, latest.as_single().unwrap()]); + for api in [ + "checkout", + "diff", + "revert_to", + "import", + "import_map", + "import_batch", + "import_batch_multiple", + "import_json", + "detached_import", + "fork_at", + "find_id_spans_between", + "frontiers_to_vv", + "vv_to_frontiers", + "minimize_frontiers", + "travel_change_ancestors", + "get_change_at", + "get_change_at_lamport", + "get_remote_change_at", + "get_changed_containers_in", + "cmp_frontiers", + "export_updates", + "export_updates_in_range", + "export_shallow", + "export_state_only", + "export_snapshot_at", + ] { + let doc = LoroDoc::new(); + doc.import(&forged).unwrap(); + if api == "detached_import" { + doc.detach(); + } + let value = doc.get_deep_value(); + let frontiers = doc.state_frontiers(); + let old_vv = doc.oplog_vv(); + let detached = doc.is_detached(); + let mut visits = 0; + let result: Result<(), String> = match api { + "checkout" => doc.checkout(&mid).map_err(|e| e.to_string()), + "diff" => doc + .diff(&mid, &latest) + .map(|_| ()) + .map_err(|e| e.to_string()), + "revert_to" => doc.revert_to(&mid).map_err(|e| e.to_string()), + "import" => doc.import(&updates).map(|_| ()).map_err(|e| e.to_string()), + "import_map" => doc + .import(&map_updates) + .map(|_| ()) + .map_err(|e| e.to_string()), + "detached_import" => doc + .import(&dependent_updates) + .map(|_| ()) + .map_err(|e| e.to_string()), + "import_batch" => doc + .import_batch(std::slice::from_ref(&updates)) + .map(|_| ()) + .map_err(|e| e.to_string()), + "import_batch_multiple" => doc + .import_batch(&[updates.clone(), map_updates.clone()]) + .map(|_| ()) + .map_err(|e| e.to_string()), + "import_json" => doc + .import_json_updates(json.clone()) + .map(|_| ()) + .map_err(|e| e.to_string()), + "fork_at" => doc.fork_at(&mid).map(|_| ()).map_err(|e| e.to_string()), + "find_id_spans_between" => doc + .try_find_id_spans_between(&mid, &latest) + .map(|_| ()) + .map_err(|e| e.to_string()), + "frontiers_to_vv" => doc + .try_frontiers_to_vv(&mid) + .map(|_| ()) + .map_err(|e| e.to_string()), + "vv_to_frontiers" => doc + .try_vv_to_frontiers(&vv) + .map(|_| ()) + .map_err(|e| e.to_string()), + "minimize_frontiers" => doc + .try_minimize_frontiers(&redundant) + .map(|_| ()) + .map_err(|e| e.to_string()), + "travel_change_ancestors" => doc + .travel_change_ancestors(&[id], &mut |_| { + visits += 1; + std::ops::ControlFlow::Continue(()) + }) + .map_err(|e| e.to_string()), + "get_change_at" => doc + .try_get_change(id) + .map(|_| ()) + .map_err(|e| e.to_string()), + "get_change_at_lamport" => doc + .with_oplog(|oplog| oplog.try_get_change_with_lamport_lte(1, id.counter as u32)) + .map(|_| ()) + .map_err(|e| e.to_string()), + "get_remote_change_at" => doc + .with_oplog(|oplog| oplog.try_get_remote_change_at(id)) + .map(|_| ()) + .map_err(|e| e.to_string()), + "get_changed_containers_in" => doc + .try_get_changed_containers_in(id, 1) + .map(|_| ()) + .map_err(|e| e.to_string()), + "cmp_frontiers" => doc + .cmp_frontiers(&mid, &latest) + .map(|_| ()) + .map_err(|e| e.to_string()), + "export_updates" => doc + .export(ExportMode::all_updates()) + .map(|_| ()) + .map_err(|e| e.to_string()), + "export_updates_in_range" => doc + .export(ExportMode::updates_in_range(vec![loro::IdSpan::new( + 1, + id.counter, + *source.oplog_vv().get(&1).unwrap(), + )])) + .map(|_| ()) + .map_err(|e| e.to_string()), + "export_shallow" => doc + .export(ExportMode::shallow_snapshot(&mid)) + .map(|_| ()) + .map_err(|e| e.to_string()), + "export_state_only" => doc + .export(ExportMode::state_only(Some(&mid))) + .map(|_| ()) + .map_err(|e| e.to_string()), + "export_snapshot_at" => doc + .export(ExportMode::snapshot_at(&mid)) + .map(|_| ()) + .map_err(|e| e.to_string()), + _ => unreachable!(), + }; + let err = result.unwrap_err(); + assert!( + err.contains("cannot parse change block"), + "{api}, block={block}: {err}" + ); + assert_eq!(doc.get_deep_value(), value, "{api}, block={block}"); + assert_eq!(doc.state_frontiers(), frontiers, "{api}, block={block}"); + assert_eq!(doc.oplog_vv(), old_vv, "{api}, block={block}"); + assert_eq!(doc.is_detached(), detached, "{api}, block={block}"); + assert_eq!(visits, 0); + // Locks are usable and every subsequent fallible reader sees the record. + assert!(doc + .checkout(&latest) + .unwrap_err() + .to_string() + .contains("cannot parse change block")); + } + } + // Initial shallow import itself needs the root's DAG node, unlike a full + // snapshot import. Its first unreadable block must also roll back cleanly. + let root = Frontiers::from(ID::new(1, 38)); + let shallow = source.export(ExportMode::shallow_snapshot(&root)).unwrap(); + let (forged, _) = snapshot_with_truncated_history(&shallow, 0); + let doc = LoroDoc::new(); + let value = doc.get_deep_value(); + let err = doc.import(&forged).unwrap_err(); + assert!(err.to_string().contains("cannot parse change block")); + assert_eq!(doc.get_deep_value(), value); + assert!(doc.oplog_vv().is_empty()); + assert!(!doc.is_detached()); + doc.import(&snapshot).unwrap(); + assert_eq!(doc.get_deep_value(), source.get_deep_value()); +}