diff --git a/benches/deserialize.rs b/benches/deserialize.rs index bd024603..6d88bc4f 100644 --- a/benches/deserialize.rs +++ b/benches/deserialize.rs @@ -4,6 +4,9 @@ use clvmr::serde::{ node_to_bytes_backrefs, serialized_length_from_bytes, serialized_length_from_bytes_trusted, tree_hash_from_stream, }; +use clvmr::serde_2026::{deserialize_2026_body, serialize_2026}; + +const BENCH_MAX_ATOM_LEN: usize = 1 << 20; use criterion::{Criterion, criterion_group, criterion_main}; use std::include_bytes; use std::time::Instant; @@ -76,6 +79,34 @@ fn deserialize_benchmark(c: &mut Criterion) { }); } + { + let mut a = Allocator::new(); + let node = node_from_bytes(&mut a, block).expect("node_from_bytes"); + let serialized_2026 = serialize_2026(&a, node).expect("serialize_2026"); + + let mut a = Allocator::new(); + let node = node_from_bytes_backrefs(&mut a, compressed_block.as_ref()) + .expect("node_from_bytes_backrefs"); + let serialized_2026_compressed = serialize_2026(&a, node).expect("serialize_2026"); + + for (data, name_suffix) in &[ + (serialized_2026.as_slice(), ""), + (serialized_2026_compressed.as_slice(), "-compressed-src"), + ] { + let mut a = Allocator::new(); + let iter_checkpoint = a.checkpoint(); + group.bench_function(format!("deserialize_2026_body{name_suffix}"), |b| { + b.iter(|| { + a.restore_checkpoint(&iter_checkpoint); + let start = Instant::now(); + deserialize_2026_body(&mut a, data, BENCH_MAX_ATOM_LEN, false) + .expect("deserialize_2026_body"); + start.elapsed() + }) + }); + } + } + let mut a = Allocator::new(); let iter_checkpoint = a.checkpoint(); group.bench_function("node_from_bytes", |b| { diff --git a/benches/serialize.rs b/benches/serialize.rs index 6e017b40..c3093f42 100644 --- a/benches/serialize.rs +++ b/benches/serialize.rs @@ -3,6 +3,7 @@ use clvmr::serde::{ Serializer, node_from_bytes, node_from_bytes_backrefs, node_to_bytes_backrefs, node_to_bytes_limit, }; +use clvmr::serde_2026::serialize_2026; use criterion::black_box; use criterion::{Criterion, criterion_group, criterion_main}; use std::include_bytes; @@ -56,6 +57,14 @@ fn serialize_benchmark(c: &mut Criterion) { start.elapsed() }) }); + + group.bench_function(format!("serialize_2026 {name}"), |b| { + b.iter(|| { + let start = Instant::now(); + black_box(serialize_2026(&a, node).expect("serialize_2026")); + start.elapsed() + }) + }); } group.finish(); diff --git a/docs/serde-2026.md b/docs/serde-2026.md new file mode 100644 index 00000000..3cac4acb --- /dev/null +++ b/docs/serde-2026.md @@ -0,0 +1,145 @@ +# 2026 Serialization Format + +## Magic Header + +Every 2026-format blob begins with a 6-byte magic prefix: + +``` +0xfd 0xff 0x32 0x30 0x32 0x36 +``` + +Rationale: + +- `0xfd 0xff` drives legacy/backrefs decoders down an invalid atom-size path, + causing immediate failure. +- `0x32 0x30 0x32 0x36` is ASCII `"2026"` for readable hexdumps. + +### Detection + +The magic prefix allows helper APIs to distinguish 2026-format blobs from +legacy/backref blobs: + +- If the blob starts with `0xfd 0xff 0x32 0x30 0x32 0x36`, it is 2026-format. +- Otherwise, parse with the legacy/backrefs path. + +Consensus callers do not need to rely on auto-detection. They can select the +expected format from fork height or consensus flags and call the corresponding +deserializer directly. + +### Backward Compatibility + +When a 2026-format blob is handed to a legacy deserializer unaware of the new +format, it should fail quickly due to the deliberately invalid size prefix. + +## Payload Format + +After the 6-byte magic header, the payload consists of two sections: + +1. **Atom table** — all unique atoms (except nil), grouped by length +2. **Instruction stream** — stack-based operations to reconstruct the tree + +### Atom Table + +Nil (the empty atom) is **not** included in the atom table — it has a dedicated +opcode (`0`) in the instruction stream. + +The atom table begins with a varint encoding the number of atom groups. + +For each group (in stream order): + +- If the group contains **one** atom: a positive varint encoding the atom's byte + length, followed by the atom's raw bytes. +- If the group contains **multiple** atoms of the same length: a negative varint + encoding the negated byte length, then a positive varint encoding the count, + then the raw bytes of each atom concatenated (each is exactly `length` bytes). + +Atom lengths must be non-zero because nil is excluded from the atom table. +Deserializers enforce a configurable maximum atom length (default: 1 MiB) and a +maximum input byte budget (default: 10 MiB). Separate atom-group, atom-count, +instruction-count, stack-size, and pair-count limits are not needed for DoS +protection: every declared item must consume at least one input byte before it +can produce parser work or allocate a CLVM node. The input byte budget therefore +bounds all of those quantities. + +Atoms are assigned indices starting from 0, in the order they appear in the +table. + +The decoder accepts groups in any order. Multiple groups with the same byte +length are valid (they contribute separate atom indices). A serializer may +choose a specific ordering strategy (for example, sorting by frequency so +commonly-referenced atoms land in lower index ranges whose varint encodings are +shorter). + +### Instruction Stream + +The instruction stream begins with a varint encoding the total number of +instructions. + +Each instruction is a varint: + +| Varint value | Meaning | +| ------------ | ---------------------------------------------------------------- | +| `0` | Push nil (the empty atom) | +| `1` | Pop two items (left was pushed first), cons them, push the pair | +| `-1` | Pop two items (right was pushed first), cons them, push the pair | +| `N >= 2` | Push the atom at index N-2 onto the stack | +| `N <= -2` | Push the already-constructed pair at index -N-2 onto the stack | + +Pairs are indexed in construction order (the first pair cons'd is index 0, the +second is index 1, etc.). A negative instruction references a pair that was +previously constructed during this same decode, enabling shared sub-trees +without re-encoding them. + +The current serializer emits left-first cons instructions (`1`). Decoders accept +right-first cons instructions (`-1`) so future serializers can choose different +pair visit orders without changing the wire format. + +After all instructions execute, the stack must contain exactly one item: the +root node. + +### Varint Encoding + +Signed integers are encoded with a variable-length prefix scheme: + +``` +0xxxxxxx → 7-bit value, range [-64, 63] +10xxxxxx xxxxxxxx → 14-bit value, range [-8192, 8191] +110xxxxx xxxxxxxx xxxxxxxx → 21-bit value, range [-1048576, 1048575] +1110xxxx xxxxxxxx xxxxxxxx xxxxxxxx + → 28-bit value +11110xxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + → 35-bit value +111110xx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + → 42-bit value +1111110x xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + → 49-bit value +11111110 xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + → 56-bit value +``` + +The number of leading `1` bits determines how many additional bytes follow, +similar to UTF-8 prefix-length coding. A `0` separator bit follows the leading +`1`s. The remaining bits (across all bytes) form a two's-complement signed +integer in big-endian order. + +A prefix of 8 leading `1` bits (`0xFF`) is invalid. + +The deserializer has a `strict` mode that rejects overlong varint encodings. In +strict mode, every varint must use the shortest encoding that can represent its +value. Lenient mode accepts overlong encodings for tooling/backward-compatible +parsing. + +## Size Bound + +For the current instruction-stream format, the analysis in +`generator-identity-hf-analysis/docs/SERDE2026_UPPER_BOUND.md` proves: + +``` +serde_2026_bytes <= atom_bytes + 2 * unique_atoms + 3 * unique_pairs + 5 +``` + +assuming all atom lengths fit in a 4-byte varint (`length <= 2^27 - 1`). This +condition is far weaker than the default 1 MiB atom limit. Because the hard fork +cost formula charges this same size component, consensus callers can derive +their accepted serde_2026 byte budget from the 11B block cost limit instead of +choosing an arbitrary message-size cap. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index a5ecbdfc..385d9871 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -148,3 +148,15 @@ name = "modpow" path = "fuzz_targets/modpow.rs" test = false doc = false + +[[bin]] +name = "serde-2026" +path = "fuzz_targets/serde_2026.rs" +test = false +doc = false + +[[bin]] +name = "serde-2026-varint" +path = "fuzz_targets/serde_2026_varint.rs" +test = false +doc = false diff --git a/fuzz/fuzz_targets/serde_2026.rs b/fuzz/fuzz_targets/serde_2026.rs new file mode 100644 index 00000000..fb5537de --- /dev/null +++ b/fuzz/fuzz_targets/serde_2026.rs @@ -0,0 +1,69 @@ +#![no_main] + +use clvm_fuzzing::ArbitraryClvmTree; +use clvmr::serde::node_to_bytes; +use clvmr::serde_2026::{deserialize_2026_body, serialize_2026_level}; +use clvmr::{Allocator, allocator::NodePtr}; +use libfuzzer_sys::{Corpus, fuzz_target}; + +const FUZZ_MAX_ATOM_LEN: usize = 1 << 20; + +#[derive(arbitrary::Arbitrary, Debug)] +enum FuzzInput { + Bytes(Vec), + Tree(Box>), +} + +fn canonical(a: &Allocator, node: NodePtr) -> Vec { + node_to_bytes(a, node).expect("node_to_bytes failed") +} + +fn roundtrip_check(label: &str, a: &Allocator, original: NodePtr, blob: &[u8]) { + let mut a2 = Allocator::new(); + let decoded = deserialize_2026_body(&mut a2, blob, FUZZ_MAX_ATOM_LEN, false) + .unwrap_or_else(|e| panic!("{label}: deserialize failed: {e:?}")); + assert_eq!( + canonical(a, original), + canonical(&a2, decoded), + "{label}: tree mismatch" + ); +} + +fn check_tree(a: &Allocator, node: NodePtr) { + for (label, level) in serialization_strategies() { + let blob = + serialize_2026_level(a, node, level).unwrap_or_else(|_| panic!("{label} failed")); + roundtrip_check(label, a, node, &blob); + } +} + +fn serialization_strategies() -> impl Iterator { + std::iter::once(("fast", 0)) +} + +fuzz_target!(|input: FuzzInput| -> Corpus { + match input { + FuzzInput::Bytes(data) => { + let mut a = Allocator::new(); + let Ok(node) = deserialize_2026_body(&mut a, &data, FUZZ_MAX_ATOM_LEN, false) else { + return Corpus::Reject; + }; + check_tree(&a, node); + } + FuzzInput::Tree(program) => { + check_tree(&program.allocator, program.tree); + + let mut a2 = Allocator::new(); + let blob = + serialize_2026_level(&program.allocator, program.tree, 0).expect("Fast failed"); + let decoded = deserialize_2026_body(&mut a2, &blob, FUZZ_MAX_ATOM_LEN, false) + .expect("deserialize fast failed"); + assert_eq!( + canonical(&program.allocator, program.tree), + canonical(&a2, decoded) + ); + } + } + + Corpus::Keep +}); diff --git a/fuzz/fuzz_targets/serde_2026_varint.rs b/fuzz/fuzz_targets/serde_2026_varint.rs new file mode 100644 index 00000000..62aaef55 --- /dev/null +++ b/fuzz/fuzz_targets/serde_2026_varint.rs @@ -0,0 +1,46 @@ +#![no_main] + +use clvmr::serde_2026::{decode_varint, encode_varint}; +use libfuzzer_sys::fuzz_target; +use std::io::Cursor; + +fuzz_target!(|data: &[u8]| { + // Non-strict decode must never panic on arbitrary input. + let mut cur = Cursor::new(data); + let non_strict = decode_varint(&mut cur, false); + + if let Ok(v) = non_strict { + let consumed = cur.position() as usize; + + // The canonical (minimal) encoding must: + // - decode in strict mode to the same value + // - never be longer than the bytes we just consumed + // (strict-mode rejections are exactly the "overlong" case) + let canonical = encode_varint(v); + assert!( + canonical.len() <= consumed, + "canonical encoding of {v} is {} bytes but input took {consumed}", + canonical.len(), + ); + let strict_canonical = decode_varint(&mut Cursor::new(&canonical[..]), true) + .expect("canonical encoding must decode under strict"); + assert_eq!(v, strict_canonical, "canonical roundtrip mismatch"); + + // Strict-mode decode of the original input either matches the + // non-strict value (canonical input) or errors (overlong / non-canonical). + match decode_varint(&mut Cursor::new(data), true) { + Ok(strict_v) => assert_eq!(strict_v, v, "strict and non-strict disagree"), + Err(_) => { + // Overlong-but-valid-as-non-strict encoding. Confirm it really is + // non-canonical: the canonical form must be shorter. + assert!( + canonical.len() < consumed, + "strict rejected a canonical-length encoding" + ); + } + } + } else { + // If non-strict failed, strict must also fail (strict is stricter). + assert!(decode_varint(&mut Cursor::new(data), true).is_err()); + } +}); diff --git a/src/lib.rs b/src/lib.rs index b0c582cb..d89ed6bc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod run_program; pub mod runtime_dialect; pub mod secp_ops; pub mod serde; +pub mod serde_2026; pub mod sha_tree_op; pub mod traverse_path; pub mod treehash; diff --git a/src/serde/mod.rs b/src/serde/mod.rs index 4320eaf8..0dea4c0d 100644 --- a/src/serde/mod.rs +++ b/src/serde/mod.rs @@ -41,3 +41,10 @@ pub use tools::{ tree_hash_from_stream, }; pub use tree_cache::{TreeCache, TreeCacheCheckpoint}; + +pub use crate::serde_2026::{ + SERDE_2026_MAGIC_PREFIX, deserialize_2026_body, deserialize_2026_body_from_stream, + node_from_bytes_serde_2026, node_to_bytes_serde_2026, node_to_bytes_serde_2026_level, + serialize_2026, serialize_2026_level, serialize_2026_to_stream, serialize_2026_to_stream_level, + serialized_length_serde_2026, +}; diff --git a/src/serde_2026/de.rs b/src/serde_2026/de.rs new file mode 100644 index 00000000..284bacb5 --- /dev/null +++ b/src/serde_2026/de.rs @@ -0,0 +1,232 @@ +use std::io::{Cursor, Read}; + +use crate::allocator::{Allocator, NodePtr}; +use crate::error::{EvalErr, Result}; + +use super::SERDE_2026_MAGIC_PREFIX; +use super::varint::decode_varint; + +fn checked_usize(value: i64) -> Result { + if value < 0 { + return Err(EvalErr::SerializationError); + } + usize::try_from(value).map_err(|_| EvalErr::SerializationError) +} + +fn checked_bounded_usize(value: i64, max: usize) -> Result { + let value = checked_usize(value)?; + if value > max { + return Err(EvalErr::SerializationError); + } + Ok(value) +} + +/// Deserialize a node from a stream using the 2026 format. +/// +/// **Reads the body only — does *not* expect the magic prefix.** Callers +/// that have a full prefix-framed blob should use [`node_from_bytes_serde_2026`] +/// (slice) or strip the prefix themselves before calling this. +/// +/// `max_atom_len` caps the byte length of any single atom (and therefore the +/// pre-allocation done while reading atom bytes). `strict` rejects overlong / +/// non-minimal varint encodings. +/// +/// **Caller contract:** `reader` must be bounded — for example via +/// [`std::io::Read::take`] — otherwise a malformed blob can drive an unbounded +/// loop. Total input policy belongs in the caller, not here. Use +/// [`deserialize_2026_body`] when you have a slice; the slice's length is the +/// natural bound. +pub fn deserialize_2026_body_from_stream( + allocator: &mut Allocator, + reader: &mut R, + max_atom_len: usize, + strict: bool, +) -> Result { + let mut atoms: Vec = Vec::new(); + let group_count = checked_usize(decode_varint(reader, strict)?)?; + let mut buf: Vec = Vec::new(); + + for _ in 0..group_count { + let length_val = decode_varint(reader, strict)?; + let (length, count) = if length_val < 0 { + if length_val == i64::MIN { + return Err(EvalErr::SerializationError); + } + ( + checked_bounded_usize(-length_val, max_atom_len)?, + checked_usize(decode_varint(reader, strict)?)?, + ) + } else { + (checked_bounded_usize(length_val, max_atom_len)?, 1) + }; + if length == 0 || count == 0 { + return Err(EvalErr::SerializationError); + } + buf.resize(length, 0); + for _ in 0..count { + reader + .read_exact(&mut buf) + .map_err(|_| EvalErr::SerializationError)?; + atoms.push(allocator.new_atom(&buf)?); + } + } + + let instruction_count = checked_usize(decode_varint(reader, strict)?)?; + if instruction_count == 0 { + return Err(EvalErr::SerializationError); + } + + let nil = allocator.nil(); + // Don't pre-size `pairs` from `instruction_count`: a tiny blob could + // declare ~2^54 instructions and trick `Vec::with_capacity` into a + // multi-PB allocation request that aborts the process. The bounded reader + // (slice length, or caller-supplied `Take`) terminates the loop long + // before we reach a pathological size; geometric growth from `Vec::new()` + // handles the typical case in a handful of reallocs. + let mut pairs: Vec = Vec::new(); + let mut stack: Vec = Vec::with_capacity(64); + + for _ in 0..instruction_count { + let inst = decode_varint(reader, strict)?; + match inst { + 0 => stack.push(nil), + 1 => { + if stack.len() < 2 { + return Err(EvalErr::SerializationError); + } + let right = stack.pop().unwrap(); + let left = stack.pop().unwrap(); + let pair = allocator.new_pair(left, right)?; + pairs.push(pair); + stack.push(pair); + } + -1 => { + if stack.len() < 2 { + return Err(EvalErr::SerializationError); + } + let left = stack.pop().unwrap(); + let right = stack.pop().unwrap(); + let pair = allocator.new_pair(left, right)?; + pairs.push(pair); + stack.push(pair); + } + n if n >= 2 => { + let ai = (n - 2) as usize; + stack.push(*atoms.get(ai).ok_or(EvalErr::SerializationError)?); + } + n => { + let pi = (-n - 2) as usize; + stack.push(*pairs.get(pi).ok_or(EvalErr::SerializationError)?); + } + } + } + + if stack.len() != 1 { + return Err(EvalErr::SerializationError); + } + Ok(stack[0]) +} + +/// Deserialize a node from a byte slice using the 2026 format. +/// +/// **Body only — does *not* strip the magic prefix.** Pairs with +/// [`serialize_2026`]. Callers with a prefix-framed blob should use +/// [`node_from_bytes_serde_2026`]. +/// +/// The slice's length naturally bounds the input; no separate input-byte +/// budget is required. `max_atom_len` caps the byte length of any single +/// atom and `strict` rejects overlong / non-minimal varint encodings. +pub fn deserialize_2026_body( + allocator: &mut Allocator, + data: &[u8], + max_atom_len: usize, + strict: bool, +) -> Result { + deserialize_2026_body_from_stream(allocator, &mut Cursor::new(data), max_atom_len, strict) +} + +/// Deserialize a magic-prefixed serde_2026 blob. +/// +/// Verifies and strips [`SERDE_2026_MAGIC_PREFIX`], then delegates to +/// [`deserialize_2026_body`]. Pairs with [`super::ser::node_to_bytes_serde_2026`]. +pub fn node_from_bytes_serde_2026( + allocator: &mut Allocator, + blob: &[u8], + max_atom_len: usize, + strict: bool, +) -> Result { + let body = blob + .strip_prefix(SERDE_2026_MAGIC_PREFIX.as_slice()) + .ok_or(EvalErr::SerializationError)?; + deserialize_2026_body(allocator, body, max_atom_len, strict) +} + +/// Compute the serialized length of a serde_2026 blob (including magic prefix). +/// +/// Walks the header structure without allocating or building a CLVM tree. +/// Mirrors every header-time validation +/// [`deserialize_2026_body_from_stream`] performs, so a blob that returns +/// `Ok(len)` here is guaranteed to clear the deserializer's header parse +/// (instruction-stream stack invariants and index validity are still checked +/// at deserialize time — those can't be byte-counted). +/// +/// `max_atom_len` caps any declared atom length and `strict` rejects +/// overlong / non-minimal varint encodings — pass the same values you would +/// pass to the deserializer. For framing-only callers that don't want a +/// policy opinion, `usize::MAX` and `false` accept anything the deserializer +/// could itself parse on a sufficiently permissive caller. +/// +/// The input buffer may contain trailing data; only the bytes belonging to +/// the serde_2026 blob are counted. +pub fn serialized_length_serde_2026(buf: &[u8], max_atom_len: usize, strict: bool) -> Result { + if !buf.starts_with(&SERDE_2026_MAGIC_PREFIX) { + return Err(EvalErr::SerializationError); + } + + let data = &buf[SERDE_2026_MAGIC_PREFIX.len()..]; + let mut cursor = Cursor::new(data); + + let group_count = checked_usize(decode_varint(&mut cursor, strict)?)?; + for _ in 0..group_count { + let length_val = decode_varint(&mut cursor, strict)?; + let skip = if length_val < 0 { + if length_val == i64::MIN { + return Err(EvalErr::SerializationError); + } + let atom_len = checked_bounded_usize(-length_val, max_atom_len)?; + let count = checked_usize(decode_varint(&mut cursor, strict)?)?; + if atom_len == 0 || count == 0 { + return Err(EvalErr::SerializationError); + } + (atom_len as u64) + .checked_mul(count as u64) + .ok_or(EvalErr::SerializationError)? + } else { + let atom_len = checked_bounded_usize(length_val, max_atom_len)?; + if atom_len == 0 { + return Err(EvalErr::SerializationError); + } + atom_len as u64 + }; + let new_pos = cursor + .position() + .checked_add(skip) + .ok_or(EvalErr::SerializationError)?; + if new_pos > data.len() as u64 { + return Err(EvalErr::SerializationError); + } + cursor.set_position(new_pos); + } + + let instruction_count = checked_usize(decode_varint(&mut cursor, strict)?)?; + // Mirror `deserialize_2026_body_from_stream`: instruction_count == 0 + // leaves the stack empty and is rejected there, so reject it here too. + if instruction_count == 0 { + return Err(EvalErr::SerializationError); + } + for _ in 0..instruction_count { + decode_varint(&mut cursor, strict)?; + } + + Ok(SERDE_2026_MAGIC_PREFIX.len() as u64 + cursor.position()) +} diff --git a/src/serde_2026/mod.rs b/src/serde_2026/mod.rs new file mode 100644 index 00000000..4c841ffe --- /dev/null +++ b/src/serde_2026/mod.rs @@ -0,0 +1,62 @@ +//! 2026 Serialization Format for CLVM. +//! +//! Deduplicates atoms and pairs via interning, uses variable-length integer +//! encoding (varints), and groups atoms by length for better compression. +//! +//! ## Format Overview +//! +//! 1. Atom table: grouped by length, with varint-encoded counts (nil excluded) +//! 2. Instruction stream: stack-based operations to reconstruct the tree +//! +//! ## Instructions +//! +//! - `0`: Push nil +//! - `1`: Pop two items (left was pushed first), cons them, push result +//! - `-1`: Pop two items (right was pushed first), cons them, push result +//! - `>= 2` (positive varint N): Push atom at index N-2 +//! - `<= -2` (negative varint N): Push already-constructed pair at index -N-2 +//! +//! The current serializer always uses opcode `1` (left-first cons). The format +//! accepts `-1` so future serializers can choose right-first traversal when +//! that helps compression. + +mod de; +mod ser; +mod strategy; +mod varint; + +#[cfg(test)] +mod tests; + +/// Magic prefix bytes for serde_2026 format. +/// +/// - `0xfd 0xff` forces legacy/backref decoders down an invalid atom-length +/// path (fail-fast). +/// - `0x32 0x30 0x32 0x36` is ASCII `"2026"` for readable hexdumps. +pub const SERDE_2026_MAGIC_PREFIX: [u8; 6] = [0xfd, 0xff, b'2', b'0', b'2', b'6']; + +/// Maximum atoms/pairs that fit in i32 indices (used by instruction stream). +const MAX_INDEX: usize = i32::MAX as usize; + +/// Internal vocabulary for which serialization strategy to use. The public API +/// takes a `level: u32` and saturates to the highest implemented level; this +/// enum is the after-saturation result. +/// +/// - `Fast` (level 0): left-first traversal. O(N) serialization. +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] +#[repr(u8)] +pub(crate) enum Compression { + #[default] + Fast = 0, +} + +pub use de::{ + deserialize_2026_body, deserialize_2026_body_from_stream, node_from_bytes_serde_2026, + serialized_length_serde_2026, +}; +pub use ser::{ + node_to_bytes_serde_2026, node_to_bytes_serde_2026_level, serialize_2026, serialize_2026_level, + serialize_2026_to_stream, serialize_2026_to_stream_level, +}; +#[doc(hidden)] +pub use varint::{decode_varint, encode_varint}; diff --git a/src/serde_2026/ser.rs b/src/serde_2026/ser.rs new file mode 100644 index 00000000..657d34be --- /dev/null +++ b/src/serde_2026/ser.rs @@ -0,0 +1,377 @@ +use std::collections::HashMap; +use std::io::Write; + +use crate::allocator::{Allocator, NodePtr, SExp}; +use crate::error::{EvalErr, Result}; +use crate::serde::{InternedTree, intern_tree}; + +use super::MAX_INDEX; +use super::strategy::{Direction, LeftFirst, VisitStrategy}; +use super::varint::write_varint; + +/// Intermediate state after interning and sorting atoms. +/// +/// Serialization strategies share this prep work. +pub(super) struct SerializerState { + pub tree: InternedTree, + pub sorted_no_nil: Vec, + pub atom_remap: HashMap, + pub nil_old_idx: Option, + pub pairs: Vec<(i32, i32)>, + pub root_index: i32, +} + +/// Count how many times each atom is referenced (by root or as a pair child). +fn atom_ref_counts( + tree: &InternedTree, + node_to_index: &impl Fn(NodePtr) -> i32, + root_index: i32, +) -> Vec { + let mut counts = vec![0u32; tree.atoms.len()]; + if root_index >= 0 { + counts[root_index as usize] += 1; + } + for &pair_node in &tree.pairs { + let (left, right) = match tree.allocator.sexp(pair_node) { + SExp::Pair(l, r) => (l, r), + _ => unreachable!(), + }; + for child in [left, right] { + let idx = node_to_index(child); + if idx >= 0 { + counts[idx as usize] += 1; + } + } + } + counts +} + +/// Sort atoms by (reused-first, frequency desc, shorter first, stable). +/// Returns (sorted indices excluding nil, old->new index remap, nil's old index). +fn sort_atoms( + tree: &InternedTree, + ref_counts: &[u32], +) -> (Vec, HashMap, Option) { + let atom_count = tree.atoms.len(); + + let mut sorted: Vec = (0..atom_count).collect(); + sorted.sort_by(|&a, &b| { + let a_reused = ref_counts[a] > 1; + let b_reused = ref_counts[b] > 1; + b_reused + .cmp(&a_reused) + .then_with(|| ref_counts[b].cmp(&ref_counts[a])) + .then_with(|| { + tree.allocator + .atom_len(tree.atoms[a]) + .cmp(&tree.allocator.atom_len(tree.atoms[b])) + }) + .then_with(|| a.cmp(&b)) + }); + + let nil_old_idx: Option = tree + .atoms + .iter() + .position(|&a| tree.allocator.atom_len(a) == 0) + .map(|i| i as i32); + + let sorted_no_nil: Vec = sorted + .iter() + .filter(|&&old_idx| Some(old_idx as i32) != nil_old_idx) + .copied() + .collect(); + + let mut atom_remap = HashMap::with_capacity(sorted_no_nil.len()); + for (new_idx, &old_idx) in sorted_no_nil.iter().enumerate() { + atom_remap.insert(old_idx as i32, new_idx as i32); + } + + (sorted_no_nil, atom_remap, nil_old_idx) +} + +impl SerializerState { + pub fn new(allocator: &Allocator, node: NodePtr) -> Result { + let tree = intern_tree(allocator, node)?; + + if tree.atoms.len() > MAX_INDEX || tree.pairs.len() > MAX_INDEX { + return Err(EvalErr::SerializationError); + } + + let mut atom_to_index = HashMap::with_capacity(tree.atoms.len()); + for (i, &atom) in tree.atoms.iter().enumerate() { + atom_to_index.insert(atom, i as i32); + } + let mut pair_to_index = HashMap::with_capacity(tree.pairs.len()); + for (i, &pair) in tree.pairs.iter().enumerate() { + pair_to_index.insert(pair, -(i as i32 + 1)); + } + + let node_to_index = |n: NodePtr| -> i32 { + if let Some(&idx) = atom_to_index.get(&n) { + idx + } else { + pair_to_index[&n] + } + }; + + let root_index = node_to_index(tree.root); + let ref_counts = atom_ref_counts(&tree, &node_to_index, root_index); + let (sorted_no_nil, atom_remap, nil_old_idx) = sort_atoms(&tree, &ref_counts); + + let pairs: Vec<(i32, i32)> = tree + .pairs + .iter() + .map(|&pair_node| { + let (left, right) = match tree.allocator.sexp(pair_node) { + SExp::Pair(l, r) => (l, r), + _ => unreachable!(), + }; + (node_to_index(left), node_to_index(right)) + }) + .collect(); + + Ok(Self { + tree, + sorted_no_nil, + atom_remap, + nil_old_idx, + pairs, + root_index, + }) + } +} + +/// Write the atom table (nil excluded) grouped by contiguous equal lengths. +pub(super) fn write_atom_table( + writer: &mut W, + tree: &InternedTree, + sorted_no_nil: &[usize], +) -> Result<()> { + let mut atom_groups: Vec<(usize, Vec)> = Vec::new(); + for &old_idx in sorted_no_nil { + let atom_node = tree.atoms[old_idx]; + let len = tree.allocator.atom_len(atom_node); + if let Some((last_len, atoms)) = atom_groups.last_mut() + && *last_len == len + { + atoms.push(atom_node); + } else { + atom_groups.push((len, vec![atom_node])); + } + } + + write_varint(writer, atom_groups.len() as i64)?; + for (length, atoms_of_length) in &atom_groups { + if atoms_of_length.len() == 1 { + write_varint(writer, *length as i64)?; + writer.write_all(tree.allocator.atom(atoms_of_length[0]).as_ref())?; + } else { + write_varint(writer, -(*length as i64))?; + write_varint(writer, atoms_of_length.len() as i64)?; + for &atom_node in atoms_of_length { + writer.write_all(tree.allocator.atom(atom_node).as_ref())?; + } + } + } + Ok(()) +} + +/// Walk the interned pair tree under `strategy`, emitting an instruction +/// stream. Only the visit order at each pair varies across strategies. +pub(super) fn emit_instructions( + state: &SerializerState, + strategy: &S, +) -> Vec { + if state.tree.pairs.is_empty() { + let mut instructions = Vec::with_capacity(1); + if Some(state.root_index) == state.nil_old_idx { + instructions.push(0); + } else { + instructions.push(state.atom_remap[&state.root_index] as i64 + 2); + } + return instructions; + } + + enum Op { + Build(i32, C), + Cons(i32, Direction), + } + + let mut work_stack: Vec> = + vec![Op::Build(state.root_index, strategy.root_ctx(state))]; + let mut construction_order: HashMap = HashMap::new(); + let mut instructions: Vec = Vec::new(); + + while let Some(op) = work_stack.pop() { + match op { + Op::Cons(pi, dir) => { + instructions.push(dir.cons_opcode()); + construction_order.insert(pi, construction_order.len() as i32); + } + Op::Build(idx, ctx) => { + if idx >= 0 { + if Some(idx) == state.nil_old_idx { + instructions.push(0); + } else { + instructions.push(state.atom_remap[&idx] as i64 + 2); + } + } else if let Some(&ci) = construction_order.get(&idx) { + instructions.push(-(ci as i64 + 2)); + } else { + let pi = (-idx - 1) as usize; + let (left, right) = state.pairs[pi]; + let (dir, l_ctx, r_ctx) = strategy.decide(state, pi, ctx); + work_stack.push(Op::Cons(idx, dir)); + match dir { + Direction::LeftFirst => { + work_stack.push(Op::Build(right, r_ctx)); + work_stack.push(Op::Build(left, l_ctx)); + } + Direction::RightFirst => { + work_stack.push(Op::Build(left, l_ctx)); + work_stack.push(Op::Build(right, r_ctx)); + } + } + } + } + } + } + + instructions +} + +/// Write `state` to `writer` using `strategy` for pair visit order. +pub(super) fn serialize_with_strategy( + state: &SerializerState, + strategy: &S, + writer: &mut W, +) -> Result<()> { + write_atom_table(writer, &state.tree, &state.sorted_no_nil)?; + let instructions = emit_instructions(state, strategy); + write_varint(writer, instructions.len() as i64)?; + for inst in instructions { + write_varint(writer, inst)?; + } + Ok(()) +} + +/// Debug-only: deserialize `bytes` and verify it equals `node`. Panics on mismatch. +#[cfg(debug_assertions)] +pub(super) fn debug_assert_roundtrip(allocator: &Allocator, node: NodePtr, bytes: &[u8]) { + use super::de::deserialize_2026_body; + let mut probe = Allocator::new(); + // Self-check uses 1 MiB max atom (matches the legacy non-consensus default). + let decoded = deserialize_2026_body(&mut probe, bytes, 1 << 20, false) + .expect("serde_2026 self-check: produced bytes that fail to deserialize"); + assert!( + cross_allocator_eq(allocator, node, &probe, decoded), + "serde_2026 self-check: round-trip tree mismatch", + ); +} + +#[cfg(debug_assertions)] +fn cross_allocator_eq(a: &Allocator, na: NodePtr, b: &Allocator, nb: NodePtr) -> bool { + let mut stack = vec![(na, nb)]; + while let Some((x, y)) = stack.pop() { + match (a.sexp(x), b.sexp(y)) { + (SExp::Atom, SExp::Atom) => { + if a.atom(x).as_ref() != b.atom(y).as_ref() { + return false; + } + } + (SExp::Pair(xl, xr), SExp::Pair(yl, yr)) => { + stack.push((xr, yr)); + stack.push((xl, yl)); + } + _ => return false, + } + } + true +} + +// --- Public entry points --- + +use super::Compression; + +/// Map a `level: u32` onto an internal compression variant. +/// +/// Levels above the highest implemented level are *saturated* down to it, +/// so callers can pass `u32::MAX` to mean "best available compression" +/// without recompiling when new levels are added. +fn compression_for_level(_level: u32) -> Compression { + // Currently only level 0 (Fast) is implemented; every higher level + // saturates to it. As new levels land this `match` grows. + Compression::Fast +} + +fn serialize_with_compression( + allocator: &Allocator, + node: NodePtr, + compression: Compression, + writer: &mut W, +) -> Result<()> { + let state = SerializerState::new(allocator, node)?; + match compression { + Compression::Fast => serialize_with_strategy(&state, &LeftFirst, writer), + } +} + +/// Serialize a CLVM node to the 2026 format using the default level. +/// Equivalent to `serialize_2026_to_stream_level(.., 0, ..)`. +pub fn serialize_2026_to_stream( + allocator: &Allocator, + node: NodePtr, + writer: &mut W, +) -> Result<()> { + serialize_2026_to_stream_level(allocator, node, 0, writer) +} + +/// Serialize a CLVM node to the 2026 format at compression `level`. +/// +/// Levels above the highest implemented level saturate to it, so passing +/// `u32::MAX` always selects the best available compression. Currently +/// only level 0 (left-first / fast) is implemented. +pub fn serialize_2026_to_stream_level( + allocator: &Allocator, + node: NodePtr, + level: u32, + writer: &mut W, +) -> Result<()> { + serialize_with_compression(allocator, node, compression_for_level(level), writer) +} + +/// Serialize a node using the 2026 format at the default level, returning bytes. +pub fn serialize_2026(allocator: &Allocator, node: NodePtr) -> Result> { + serialize_2026_level(allocator, node, 0) +} + +/// Serialize a node using the 2026 format at compression `level`, returning bytes. +/// +/// See [`serialize_2026_to_stream_level`] for the level-saturation contract. +pub fn serialize_2026_level(allocator: &Allocator, node: NodePtr, level: u32) -> Result> { + let mut output = Vec::new(); + serialize_2026_to_stream_level(allocator, node, level, &mut output)?; + #[cfg(debug_assertions)] + debug_assert_roundtrip(allocator, node, &output); + Ok(output) +} + +/// Serialize with the magic prefix at the default level. +/// This is the recommended wire format. +pub fn node_to_bytes_serde_2026(allocator: &Allocator, node: NodePtr) -> Result> { + node_to_bytes_serde_2026_level(allocator, node, 0) +} + +/// Serialize with the magic prefix at compression `level`. +/// +/// See [`serialize_2026_to_stream_level`] for the level-saturation contract. +pub fn node_to_bytes_serde_2026_level( + allocator: &Allocator, + node: NodePtr, + level: u32, +) -> Result> { + let mut out = Vec::new(); + out.extend_from_slice(&super::SERDE_2026_MAGIC_PREFIX); + serialize_2026_to_stream_level(allocator, node, level, &mut out)?; + Ok(out) +} diff --git a/src/serde_2026/strategy.rs b/src/serde_2026/strategy.rs new file mode 100644 index 00000000..f41ae534 --- /dev/null +++ b/src/serde_2026/strategy.rs @@ -0,0 +1,60 @@ +//! Visit-order strategies for the 2026 serializer. +//! +//! The serializer walks the interned pair tree and emits a stream of +//! instructions. At each pair, the strategy decides which child to visit +//! first (`LeftFirst` -> opcode `1`, `RightFirst` -> opcode `-1`) and may +//! thread a per-node context forward (e.g. a remaining DP budget). + +use super::ser::SerializerState; + +/// Which child to visit first at a pair. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum Direction { + LeftFirst, + #[allow(dead_code)] + RightFirst, +} + +impl Direction { + pub(super) fn cons_opcode(self) -> i64 { + match self { + Direction::LeftFirst => 1, + Direction::RightFirst => -1, + } + } +} + +/// Pluggable visit-order strategy for `emit_instructions`. +/// +/// A strategy may carry its own precomputed tables (e.g. DP results) and +/// thread a per-node context (`NodeCtx`) through the traversal. +pub trait VisitStrategy { + /// Per-node context propagated down the tree. Use `()` if not needed. + type NodeCtx: Copy; + + /// Initial context for the root node. + fn root_ctx(&self, state: &SerializerState) -> Self::NodeCtx; + + /// Decide visit direction at `pair_idx` given the inherited context. + /// + /// Returns `(direction, left_ctx, right_ctx)`. + fn decide( + &self, + state: &SerializerState, + pair_idx: usize, + ctx: Self::NodeCtx, + ) -> (Direction, Self::NodeCtx, Self::NodeCtx); +} + +/// Always visit the left child first (opcode `1`). +pub struct LeftFirst; + +impl VisitStrategy for LeftFirst { + type NodeCtx = (); + + fn root_ctx(&self, _state: &SerializerState) -> Self::NodeCtx {} + + fn decide(&self, _state: &SerializerState, _pair_idx: usize, _ctx: ()) -> (Direction, (), ()) { + (Direction::LeftFirst, (), ()) + } +} diff --git a/src/serde_2026/tests.rs b/src/serde_2026/tests.rs new file mode 100644 index 00000000..c96b733b --- /dev/null +++ b/src/serde_2026/tests.rs @@ -0,0 +1,409 @@ +//! Tests for the 2026 serialization format. + +use super::{ + SERDE_2026_MAGIC_PREFIX, deserialize_2026_body, node_from_bytes_serde_2026, + node_to_bytes_serde_2026, serialize_2026_level, serialized_length_serde_2026, +}; +use crate::allocator::Allocator; +use crate::serde::{node_from_bytes_backrefs, node_to_bytes}; +use hex::FromHex; +use rstest::rstest; + +/// Sane non-consensus default for tests; the public API has no opinion. +const TEST_MAX_ATOM_LEN: usize = 1 << 20; + +// --------------------------------------------------------------------------- +// Double-round-trip, all strategies, full corpus +// --------------------------------------------------------------------------- + +/// For each legacy-hex tree: serialize with each strategy, then deserialize, +/// re-serialize, and assert identical bytes (idempotency) plus tree equivalence +/// to the original. +#[rstest] +#[case("00")] // nil +#[case("80")] // empty atom (canonical nil) +#[case("01")] // 1 +#[case("0a")] // 10 +#[case("8568656c6c6f")] // 5-byte atom "hello" +#[case("8b68656c6c6f20776f726c64")] // 11-byte atom "hello world" +#[case( + "b8400102030405060708091011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465" +)] // 64-byte atom +#[case("ff0100")] // (1 . nil) +#[case("ff0000")] // (nil . nil) +#[case("ff0101")] // (1 . 1) +#[case("ff010a")] // (1 . 10) +#[case("ff83666f6f83626172")] // (foo . bar) +#[case("ff83666f6fff8362617280")] // (foo bar) +#[case("ffff0102ff0304")] // ((1 . 2) . (3 . 4)) +#[case("ff01ff02ff03ff04ff05ff0680")] // (1 2 3 4 5 6) +#[case("ff83666f6ffe02")] // (foo . foo) +#[case("ff01ff0101")] // (1 . (1 . 1)) +#[case("ffff2a2a2a")] // ((42 . 42) . 42) +#[case("ff01ff02ff0301")] // (1 . (2 . (3 . 1))) +#[case("ff01ff02ff0300")] // (1 . (2 . (3 . nil))) +#[case("ff01ff02ff0304")] // (1 . (2 . (3 . 4))) +#[case("ff01ff02ff0103")] // (1 . (2 . (1 . 3))) +#[case("ffff0102ff0102")] // ((1 . 2) . (1 . 2)) +#[case("ffff0102ffff0102ff0102")] // ((1 . 2) . ((1 . 2) . (1 . 2))) +#[case("ffff0102ffff010200")] // ((1 . 2) . ((1 . 2) . nil)) +#[case("ffff010aff010a")] // ((1 . 10) . (1 . 10)) +#[case("ff01ff01ff0100")] // (1 . (1 . (1 . nil))) +#[case("ff01ff01ff0101")] // (1 . (1 . (1 . 1))) +#[case("ffff01ff0203ff01ff0203")] // ((1.(2.3)) . (1.(2.3))) +#[case("ffff0102ffff0102ffff010200")] // ((1.2) . ((1.2) . ((1.2) . nil))) +#[case("ff846c6f6e67ff86737472696e67ff826f66fffe0bff8474657874fffe1780")] // (long string of text) +#[case("ff83666f6ffffe01fffe01fffe01fffe01fffe01fffe0180")] // backrefs chain +fn test_round_trip(#[case] hex: &str) { + let bytes = Vec::from_hex(hex).unwrap(); + let mut allocator = Allocator::new(); + let node = node_from_bytes_backrefs(&mut allocator, &bytes).unwrap(); + let canonical = node_to_bytes(&allocator, node).unwrap(); + + let blobs: Vec<(&str, u32, Vec)> = vec![( + "fast", + 0, + serialize_2026_level(&allocator, node, 0).unwrap(), + )]; + + for (label, level, blob) in &blobs { + // First trip: tree equivalence + let mut a2 = Allocator::new(); + let n2 = deserialize_2026_body(&mut a2, blob, TEST_MAX_ATOM_LEN, false).unwrap(); + assert_eq!( + node_to_bytes(&a2, n2).unwrap(), + canonical, + "{label}: tree mismatch for {hex}" + ); + + // Second trip: serialization idempotency (same compression level) + let blob2 = serialize_2026_level(&a2, n2, *level).unwrap(); + assert_eq!( + blob, &blob2, + "{label}: double round-trip mismatch for {hex}" + ); + } +} + +// --------------------------------------------------------------------------- +// Malformed input rejection +// --------------------------------------------------------------------------- + +#[rstest] +#[case(&[0x7f], "negative atom group count")] +#[case(&[0xff], "0xFF invalid varint prefix")] +#[case(&[0x80], "truncated multibyte varint")] +#[case(&[0x80, 0x80], "truncated 3-byte varint")] +#[case(&[], "empty input")] +#[case(&[0x01, 0x01, 0x41], "valid atom table, truncated before instruction count")] +#[case(&[0x01, 0x01, 0x41, 0x02, 0x02], "instruction count=2 but only 1 instruction follows")] +#[case(&[0x01, 0x01, 0x41, 0x01, 0x70], "instruction refs atom index 110, only 1 exists")] +#[case(&[0x01, 0x01, 0x41, 0x00], "zero instructions with non-empty atom table")] +#[case(&[0x00, 0x00], "zero groups and zero instructions")] +#[case(&[0x02, 0x01, 0x41, 0x01, 0x42], "two groups claimed, only one provided")] +fn test_deserialize_rejects_malformed(#[case] data: &[u8], #[case] _desc: &str) { + let mut allocator = Allocator::new(); + assert!( + deserialize_2026_body(&mut allocator, data, TEST_MAX_ATOM_LEN, false).is_err(), + "should reject: {_desc}" + ); +} + +#[test] +fn test_strict_rejects_overlong_varints() { + let mut allocator = Allocator::new(); + + // group_count=1 encoded as two bytes, then a valid single atom payload. + let overlong_group_count = [0x80, 0x01, 0x01, b'A', 0x01, 0x02]; + assert!( + deserialize_2026_body( + &mut allocator, + &overlong_group_count, + TEST_MAX_ATOM_LEN, + true + ) + .is_err() + ); + + let decoded = deserialize_2026_body( + &mut allocator, + &overlong_group_count, + TEST_MAX_ATOM_LEN, + false, + ) + .unwrap(); + assert_eq!(allocator.atom(decoded).as_ref(), b"A"); +} + +// --------------------------------------------------------------------------- +// Magic prefix and auto-detection +// --------------------------------------------------------------------------- + +#[test] +fn test_magic_prefix() { + assert_eq!( + SERDE_2026_MAGIC_PREFIX, + [0xfd, 0xff, b'2', b'0', b'2', b'6'] + ); + + let mut allocator = Allocator::new(); + let node = allocator.new_atom(b"hello").unwrap(); + let bytes = node_to_bytes_serde_2026(&allocator, node).unwrap(); + assert!(bytes.starts_with(&SERDE_2026_MAGIC_PREFIX)); +} + +// Auto-detection (sniff the magic prefix and dispatch) is a Python-only +// convenience now, exposed by `clvm_rs.serde.deserialize(blob, "auto")`. +// See `wheel/python/tests/test_serialize.py` for coverage. + +#[test] +fn test_backrefs_decoder_rejects_serde_2026() { + let mut allocator = Allocator::new(); + let node = allocator.new_atom(b"hello").unwrap(); + let prefixed = node_to_bytes_serde_2026(&allocator, node).unwrap(); + let mut a2 = Allocator::new(); + assert!(node_from_bytes_backrefs(&mut a2, &prefixed).is_err()); +} + +// --------------------------------------------------------------------------- +// serialized_length_serde_2026 +// --------------------------------------------------------------------------- + +#[test] +fn test_serialized_length() { + let mut allocator = Allocator::new(); + + // atom + let node = allocator.new_atom(b"hello").unwrap(); + let bytes = node_to_bytes_serde_2026(&allocator, node).unwrap(); + assert_eq!( + serialized_length_serde_2026(&bytes, TEST_MAX_ATOM_LEN, false).unwrap(), + bytes.len() as u64 + ); + + // pair + let left = allocator.new_atom(b"left").unwrap(); + let right = allocator.new_atom(b"right").unwrap(); + let pair = allocator.new_pair(left, right).unwrap(); + let bytes = node_to_bytes_serde_2026(&allocator, pair).unwrap(); + assert_eq!( + serialized_length_serde_2026(&bytes, TEST_MAX_ATOM_LEN, false).unwrap(), + bytes.len() as u64 + ); + + // complex tree with shared subtrees + let a = allocator.new_atom(b"shared").unwrap(); + let p1 = allocator.new_pair(a, a).unwrap(); + let b = allocator.new_atom(b"other").unwrap(); + let p2 = allocator.new_pair(p1, b).unwrap(); + let root = allocator.new_pair(p2, p1).unwrap(); + let bytes = node_to_bytes_serde_2026(&allocator, root).unwrap(); + assert_eq!( + serialized_length_serde_2026(&bytes, TEST_MAX_ATOM_LEN, false).unwrap(), + bytes.len() as u64 + ); + + // with trailing data — length should exclude it + let mut padded = bytes.clone(); + padded.extend_from_slice(b"trailing garbage"); + assert_eq!( + serialized_length_serde_2026(&padded, TEST_MAX_ATOM_LEN, false).unwrap(), + bytes.len() as u64 + ); + + // rejects non-prefixed / empty + assert!(serialized_length_serde_2026(b"\x80", TEST_MAX_ATOM_LEN, false).is_err()); + assert!(serialized_length_serde_2026(b"", TEST_MAX_ATOM_LEN, false).is_err()); +} + +/// `serialized_length_serde_2026` must reject every header-time condition +/// that `deserialize_2026_body` rejects, so callers can use the length helper to +/// gate before deserializing without observing Ok-then-Err mismatches. +#[test] +fn test_serialized_length_rejects_what_deserialize_rejects() { + use super::varint::encode_varint; + + let mk = |group_count: i64, instruction_count: i64, atom_table: &[u8]| { + let mut blob = Vec::new(); + blob.extend_from_slice(&SERDE_2026_MAGIC_PREFIX); + blob.extend_from_slice(&encode_varint(group_count)); + blob.extend_from_slice(atom_table); + blob.extend_from_slice(&encode_varint(instruction_count)); + blob + }; + + // Cases that deserialize_2026_body rejects at header time. For each one, + // serialized_length_serde_2026 must also reject. + let cases: &[(&str, Vec)] = &[ + // instruction_count = 0 with non-empty atom table + ( + "instruction_count == 0", + mk(1, 0, &[0x01, b'A']), // one group, length=1, atom='A' + ), + // group with length = 0 + ("group length == 0", mk(1, 1, &[0x00])), + // multi-atom group with count = 0 + ( + "multi-atom group count == 0", + mk(1, 1, &[0x7f, 0x00, b'A']), // length=-1 (multi-atom), count=0 + ), + ]; + + let mut a = Allocator::new(); + for (label, blob) in cases { + // Use the prefix-aware deserializer so the asymmetry under test + // (header-time rejections) is what fails, not the magic-prefix check. + assert!( + node_from_bytes_serde_2026(&mut a, blob, TEST_MAX_ATOM_LEN, false).is_err(), + "{label}: deserialize must reject" + ); + assert!( + serialized_length_serde_2026(blob, TEST_MAX_ATOM_LEN, false).is_err(), + "{label}: serialized_length must reject (mirrors deserialize)" + ); + } +} + +// --------------------------------------------------------------------------- +// Regression test for the unbounded-capacity OOM at `de.rs:145`. A tiny blob +// (under 16 bytes) declares an `instruction_count` near the max representable +// varint (~2^54). Pre-fix, the deserializer pre-allocated `instruction_count +// / 3` `NodePtr`s — a request of about 24 PB — and the process aborted with +// "memory allocation of N bytes failed". Post-fix, the deserializer starts +// with `Vec::new()` and is bounded by the input slice (or caller-supplied +// `Read::take`), so the loop runs out of bytes long before it can drive the +// vector to a pathological size and we return `Err` cleanly. +#[test] +fn deserializer_rejects_unbounded_instruction_count() { + use super::varint::encode_varint; + + let mut blob = Vec::new(); + blob.extend_from_slice(&encode_varint(0)); // group_count = 0 + blob.extend_from_slice(&encode_varint(1_i64 << 54)); // instruction_count + assert!( + blob.len() < 16, + "PoC blob stays tiny ({} bytes)", + blob.len() + ); + + let mut a = Allocator::new(); + let result = deserialize_2026_body(&mut a, &blob, TEST_MAX_ATOM_LEN, false); + assert!( + result.is_err(), + "instruction_count must be rejected before pre-allocation" + ); +} + +#[test] +fn deserializer_rejects_unbounded_group_count() { + use super::varint::encode_varint; + + let mut blob = Vec::new(); + blob.extend_from_slice(&encode_varint(1_i64 << 54)); // group_count + + let mut a = Allocator::new(); + let result = deserialize_2026_body(&mut a, &blob, TEST_MAX_ATOM_LEN, false); + assert!( + result.is_err(), + "group_count must be rejected before pre-allocation" + ); +} + +#[test] +fn deserializer_rejects_unbounded_per_group_count() { + use super::varint::encode_varint; + + let mut blob = Vec::new(); + blob.extend_from_slice(&encode_varint(1)); // group_count = 1 + blob.extend_from_slice(&encode_varint(-3)); // multi-atom group, len=3 + blob.extend_from_slice(&encode_varint(1_i64 << 54)); // count + + let mut a = Allocator::new(); + let result = deserialize_2026_body(&mut a, &blob, TEST_MAX_ATOM_LEN, false); + assert!( + result.is_err(), + "per-group count must be rejected before pre-allocation" + ); +} + +// --------------------------------------------------------------------------- +// write_atom_table +// --------------------------------------------------------------------------- + +/// Verify the wire-level structure of `write_atom_table`: atoms of the same +/// length are emitted as a single repeated-length group (negative length +/// varint + count), and atoms of different lengths split into separate groups. +/// The exact ordering of atoms inside a length group depends on the frequency +/// sort; this test only inspects structure, not order. +#[test] +fn test_write_atom_table_groups_by_length() { + use super::ser::{SerializerState, write_atom_table}; + use super::varint::decode_varint; + use std::io::{Cursor, Read}; + + // Tree with three 3-byte atoms and one 5-byte atom — every atom is forced + // into the table (no nil, no duplicates). + let mut a = Allocator::new(); + let foo = a.new_atom(b"foo").unwrap(); + let bar = a.new_atom(b"bar").unwrap(); + let baz = a.new_atom(b"baz").unwrap(); + let hello = a.new_atom(b"hello").unwrap(); + let p = a.new_pair(foo, bar).unwrap(); + let q = a.new_pair(baz, hello).unwrap(); + let root = a.new_pair(p, q).unwrap(); + + let state = SerializerState::new(&a, root).unwrap(); + let mut buf = Vec::new(); + write_atom_table(&mut buf, &state.tree, &state.sorted_no_nil).unwrap(); + + let mut cursor = Cursor::new(&buf[..]); + let group_count = decode_varint(&mut cursor, false).unwrap(); + assert_eq!(group_count, 2, "expected 2 length-groups (3, 5)"); + + let mut total_atoms = 0usize; + let mut total_bytes = 0usize; + let mut saw_repeated_3 = false; + let mut saw_singleton_5 = false; + for _ in 0..group_count { + let length_val = decode_varint(&mut cursor, false).unwrap(); + if length_val < 0 { + // multi-atom group: -length, count, then count*length raw bytes + let len = (-length_val) as usize; + let count = decode_varint(&mut cursor, false).unwrap() as usize; + assert!(len > 0 && count > 1); + let mut bytes = vec![0u8; len * count]; + cursor.read_exact(&mut bytes).unwrap(); + total_atoms += count; + total_bytes += bytes.len(); + if len == 3 && count == 3 { + saw_repeated_3 = true; + } + } else { + // singleton group: positive length, then `length` raw bytes + let len = length_val as usize; + let mut bytes = vec![0u8; len]; + cursor.read_exact(&mut bytes).unwrap(); + total_atoms += 1; + total_bytes += bytes.len(); + if len == 5 { + saw_singleton_5 = true; + } + } + } + + assert!( + saw_repeated_3, + "expected the three 3-byte atoms to share a group" + ); + assert!( + saw_singleton_5, + "expected the 5-byte atom as a singleton group" + ); + assert_eq!(total_atoms, 4); + assert_eq!(total_bytes, 3 * 3 + 5); + assert_eq!( + cursor.position() as usize, + buf.len(), + "all bytes of the atom table should be consumed" + ); +} diff --git a/src/serde_2026/varint.rs b/src/serde_2026/varint.rs new file mode 100644 index 00000000..d24279d3 --- /dev/null +++ b/src/serde_2026/varint.rs @@ -0,0 +1,225 @@ +//! Variable-length integer encoding for the 2026 serialization format. + +use std::io::{Read, Write}; + +use crate::error::{EvalErr, Result}; + +/// Write a signed integer to `w` using variable-length encoding (varint). +/// +/// Format: [leading 1s][0 separator][two's complement value] +/// - Single byte (0 leading 1s): 0[7-bit two's complement] → range [-64, 63] +/// - Two bytes (1 leading 1): 10[14-bit two's complement] → range [-8192, 8191] +/// - Three bytes (2 leading 1s): 110[21-bit two's complement] → range [-1048576, 1048575] +/// - etc. +pub fn write_varint(w: &mut W, value: i64) -> std::io::Result<()> { + // Find the smallest encoding size that can represent the value + for leading_ones in 0..8 { + let total_value_bits = 7 + 7 * leading_ones; + + // Calculate the range this encoding can represent (two's complement) + let min_value = -(1i64 << (total_value_bits - 1)); + let max_value = (1i64 << (total_value_bits - 1)) - 1; + + // Check if value fits in this encoding + if value < min_value || value > max_value { + continue; // Need more bytes + } + + // Convert value to unsigned representation (two's complement with total_value_bits) + let unsigned_value = if value < 0 { + (value + (1i64 << total_value_bits)) as u64 + } else { + value as u64 + }; + + // Build the encoding + // First byte: [leading_ones * '1'][0][bits_in_first_byte bits of value] + let first_byte = if leading_ones > 0 { + ((1u8 << leading_ones) - 1) << (8 - leading_ones) + } else { + 0 + }; + + // Extract the high bits for the first byte (big-endian order) + let high_bits = (unsigned_value >> (leading_ones * 8)) as u8; + let first_byte = first_byte | high_bits; + + w.write_all(&[first_byte])?; + for i in (0..leading_ones).rev() { + let byte_val = (unsigned_value >> (i * 8)) as u8; + w.write_all(&[byte_val])?; + } + + return Ok(()); + } + + panic!("Value too large to encode: {}", value); +} + +fn varint_size(value: i64) -> usize { + for leading_ones in 0..8 { + let total_value_bits = 7 + 7 * leading_ones; + let min_value = -(1i64 << (total_value_bits - 1)); + let max_value = (1i64 << (total_value_bits - 1)) - 1; + if value >= min_value && value <= max_value { + return leading_ones + 1; + } + } + panic!("Value too large to encode: {}", value); +} + +/// Encode a signed integer to bytes using variable-length encoding (varint). +#[allow(dead_code)] +pub fn encode_varint(value: i64) -> Vec { + let mut buf = Vec::new(); + write_varint(&mut buf, value).unwrap(); + buf +} + +/// Decode a signed integer, optionally rejecting non-minimal encodings. +pub fn decode_varint(r: &mut R, strict: bool) -> Result { + let mut first_byte_buf = [0u8; 1]; + r.read_exact(&mut first_byte_buf) + .map_err(|_| EvalErr::SerializationError)?; + let first_byte = first_byte_buf[0]; + + // Count leading ones using leading_zeros (faster than loop) + let leading_ones = (!first_byte).leading_zeros() as usize; + + // Reject invalid prefix: 8 leading ones (e.g. 0xFF) is not a valid varint encoding + if leading_ones >= 8 { + return Err(EvalErr::SerializationError); + } + + // After leading 1s and separator 0, remaining bits are the two's complement value + let bits_in_first_byte = 7 - leading_ones; + let total_value_bits = 7 + 7 * leading_ones; + + // Extract value bits from first byte (the high bits of the value) + let value_mask = (1u8 << bits_in_first_byte) - 1; + let mut unsigned_value = (first_byte & value_mask) as u64; + + // Read additional bytes using fixed-size buffer (max 7 extra bytes) + if leading_ones > 0 { + let mut extra_bytes = [0u8; 7]; + r.read_exact(&mut extra_bytes[..leading_ones]) + .map_err(|_| EvalErr::SerializationError)?; + + for &byte in &extra_bytes[..leading_ones] { + unsigned_value = (unsigned_value << 8) | (byte as u64); + } + } + + // Convert from two's complement to signed + let sign_bit = 1u64 << (total_value_bits - 1); + let value = if unsigned_value >= sign_bit { + // Negative value: subtract 2^total_value_bits + unsigned_value as i64 - (1i64 << total_value_bits) + } else { + unsigned_value as i64 + }; + + if strict && varint_size(value) != leading_ones + 1 { + return Err(EvalErr::SerializationError); + } + + Ok(value) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn test_encode_varint() { + // Test single byte encoding + assert_eq!(encode_varint(0), vec![0x00]); + assert_eq!(encode_varint(1), vec![0x01]); + assert_eq!(encode_varint(-1), vec![0x7f]); + assert_eq!(encode_varint(63), vec![0x3f]); + assert_eq!(encode_varint(-64), vec![0x40]); + + // Test two byte encoding + assert_eq!(encode_varint(64), vec![0x80, 0x40]); + assert_eq!(encode_varint(8191), vec![0x9f, 0xff]); + assert_eq!(encode_varint(-65), vec![0xbf, 0xbf]); + assert_eq!(encode_varint(-8192), vec![0xa0, 0x00]); + } + + #[test] + fn test_decode_varint() { + assert_eq!( + decode_varint(&mut Cursor::new(&[0x00][..]), false).unwrap(), + 0 + ); + assert_eq!( + decode_varint(&mut Cursor::new(&[0x01][..]), false).unwrap(), + 1 + ); + assert_eq!( + decode_varint(&mut Cursor::new(&[0x7f][..]), false).unwrap(), + -1 + ); + assert_eq!( + decode_varint(&mut Cursor::new(&[0x80, 0x40][..]), false).unwrap(), + 64 + ); + assert_eq!( + decode_varint(&mut Cursor::new(&[0x9f, 0xff][..]), false).unwrap(), + 8191 + ); + assert_eq!( + decode_varint(&mut Cursor::new(&[0xbf, 0xbf][..]), false).unwrap(), + -65 + ); + } + + #[test] + fn test_decode_rejects_invalid_prefix() { + assert!(decode_varint(&mut Cursor::new(&[0xff][..]), false).is_err()); + assert!(decode_varint(&mut Cursor::new(&[0xff, 0x00][..]), false).is_err()); + } + + #[test] + fn test_decode_rejects_truncated_multibyte() { + // 2-byte varint (0x80 prefix) with missing second byte + assert!(decode_varint(&mut Cursor::new(&[0x80][..]), false).is_err()); + // 3-byte varint (0xc0 prefix) with only 1 extra byte + assert!(decode_varint(&mut Cursor::new(&[0xc0, 0x00][..]), false).is_err()); + // 4-byte varint (0xe0 prefix) with only 2 extra bytes + assert!(decode_varint(&mut Cursor::new(&[0xe0, 0x00, 0x00][..]), false).is_err()); + } + + #[test] + fn test_decode_empty_input() { + assert!(decode_varint(&mut Cursor::new(&[][..]), false).is_err()); + } + + #[test] + fn test_strict_decode_rejects_overlong_encodings() { + // 0 and -1 fit in a single byte, but these encodings use two bytes. + assert_eq!( + decode_varint(&mut Cursor::new(&[0x80, 0x00][..]), false).unwrap(), + 0 + ); + assert!(decode_varint(&mut Cursor::new(&[0x80, 0x00][..]), true).is_err()); + + assert_eq!( + decode_varint(&mut Cursor::new(&[0xbf, 0xff][..]), false).unwrap(), + -1 + ); + assert!(decode_varint(&mut Cursor::new(&[0xbf, 0xff][..]), true).is_err()); + } + + #[test] + fn test_roundtrip_boundary_values() { + for val in [ + 0, 1, -1, 63, -64, 64, -65, 8191, -8192, 8192, -8193, 1048575, -1048576, + ] { + let encoded = encode_varint(val); + let decoded = decode_varint(&mut Cursor::new(&encoded), false).unwrap(); + assert_eq!(val, decoded, "roundtrip failed for {val}"); + } + } +} diff --git a/wheel/python/clvm_rs/clvm_rs.pyi b/wheel/python/clvm_rs/clvm_rs.pyi index da6ce8d0..c66e67af 100644 --- a/wheel/python/clvm_rs/clvm_rs.pyi +++ b/wheel/python/clvm_rs/clvm_rs.pyi @@ -10,11 +10,41 @@ def deserialize_as_tree( ) -> Tuple[List[Tuple[int, int, int]], Optional[List[bytes]]]: ... def serialized_length(blob: bytes) -> int: ... -NO_NEG_DIV: int +# --- Deserialize functions --- +def deser_legacy(blob: bytes) -> "LazyNode": ... +def deser_backrefs(blob: bytes) -> "LazyNode": ... +def deser_2026( + blob: bytes, + *, + max_atom_len: int = ..., + strict: bool = False, +) -> "LazyNode": ... +def deser_auto( + blob: bytes, + *, + max_atom_len: int = ..., + strict: bool = False, +) -> "LazyNode": ... + +# --- Serialize functions --- +def ser_legacy(node: "LazyNode") -> bytes: ... +def ser_backrefs(node: "LazyNode") -> bytes: ... +def ser_2026( + node: "LazyNode", + *, + level: int = 0, +) -> bytes: ... + +# --- Tree conversion --- +def clvm_tree_to_lazy_node(obj: CLVMStorage) -> "LazyNode": ... + NO_UNKNOWN_OPS: int LIMIT_HEAP: int MEMPOOL_MODE: int ENABLE_SHA256_TREE: int +ENABLE_SECP_OPS: int +DISABLE_OP: int +CANONICAL_INTS: int class LazyNode(CLVMStorage): atom: Optional[bytes] diff --git a/wheel/python/clvm_rs/program.py b/wheel/python/clvm_rs/program.py index c3f6acf5..3f7a90ad 100644 --- a/wheel/python/clvm_rs/program.py +++ b/wheel/python/clvm_rs/program.py @@ -4,7 +4,12 @@ from .at import at from .casts import CastableType, to_clvm_object, int_from_bytes, int_to_bytes from .chia_dialect import CHIA_DIALECT -from .clvm_rs import run_serialized_chia_program +from .clvm_rs import ( + clvm_tree_to_lazy_node, + deser_auto, + run_serialized_chia_program, + ser_2026, +) from .clvm_storage import CLVMStorage from .clvm_tree import CLVMTree from .curry_and_treehash import CurryTreehasher @@ -13,6 +18,8 @@ from .ser import sexp_from_stream, sexp_to_stream, sexp_to_bytes from .tree_hash import sha256_treehash +SERDE_2026_MAGIC_PREFIX = b"\xfd\xff2026" + class Program(CLVMStorage): @@ -35,25 +42,46 @@ def stream(self, f: BinaryIO) -> None: sexp_to_stream(self, f) @classmethod - def from_bytes(cls, blob: bytes, calculate_tree_hash: bool = True) -> Program: - obj, cursor = cls.from_bytes_with_cursor( - blob, 0, calculate_tree_hash=calculate_tree_hash - ) - return obj + def from_bytes(cls, blob: bytes) -> Program: + """Deserialize from any CLVM format (classic, backrefs, or serde_2026).""" + return cls.wrap(deser_auto(blob)) + + @classmethod + def from_bytes_backrefs(cls, blob: bytes) -> Program: + """Deserialize classic or backrefs format only (rejects serde_2026).""" + if blob.startswith(SERDE_2026_MAGIC_PREFIX): + raise ValueError("unexpected serde_2026 format; use from_bytes() for auto-detection") + return cls.wrap(deser_auto(blob)) @classmethod def from_bytes_with_cursor( - cls, blob: bytes, cursor: int, calculate_tree_hash: bool = True + cls, blob: bytes, cursor: int ) -> Tuple[Program, int]: - tree = CLVMTree.from_bytes( - blob[cursor:], calculate_tree_hash=calculate_tree_hash - ) + tree = CLVMTree.from_bytes(blob[cursor:]) obj = cls.wrap(tree) new_cursor = len(bytes(tree)) + cursor return obj, new_cursor @classmethod - def fromhex(cls, hexstr: str) -> Program: + def from_bytes_2026(cls, blob: bytes) -> "Program": + """Deserialize from 2026 format only. + + The blob must begin with the six-byte magic prefix + ``fd ff 32 30 32 36``. + """ + if not blob.startswith(SERDE_2026_MAGIC_PREFIX): + raise ValueError( + "missing 2026 magic prefix (expected first bytes fd ff 32 30 32 36)" + ) + return cls.wrap(deser_auto(blob)) + + def to_bytes_2026(self) -> bytes: + """Serialize to 2026 format (always includes the magic prefix).""" + lazy = clvm_tree_to_lazy_node(self) + return ser_2026(lazy) + + @classmethod + def fromhex(cls, hexstr: str) -> "Program": return cls.from_bytes(bytes.fromhex(hexstr)) def __bytes__(self) -> bytes: diff --git a/wheel/python/clvm_rs/serde.py b/wheel/python/clvm_rs/serde.py new file mode 100644 index 00000000..21e9b0af --- /dev/null +++ b/wheel/python/clvm_rs/serde.py @@ -0,0 +1,103 @@ +"""Serialize and deserialize CLVM in legacy, backref, and 2026 formats. + +All functions operate on bytes or LazyNode handles backed by Rust. + + from clvm_rs.serde import deserialize, serialize + + node = deserialize(blob) # auto-detects format + out = serialize(node, "2026") # serde_2026 with magic prefix + out = serialize(node, "legacy") # classic format + out = serialize(node, "2026", level=0) # serde_2026, fast left-first traversal + +Converting a Python CLVM tree to a Rust-backed LazyNode (with interning): + + from clvm_rs.serde import clvm_tree_to_lazy_node + + lazy = clvm_tree_to_lazy_node(program) + blob = serialize(lazy, "2026") +""" + +from .clvm_rs import ( + clvm_tree_to_lazy_node, + deser_2026, + deser_auto, + deser_backrefs, + deser_legacy, + ser_2026, + ser_backrefs, + ser_legacy, +) + +__all__ = [ + "serialize", + "deserialize", + "clvm_tree_to_lazy_node", + "deser_legacy", + "deser_backrefs", + "deser_2026", + "deser_auto", + "ser_legacy", + "ser_backrefs", + "ser_2026", +] + +_DESERIALIZERS = { + "legacy": deser_legacy, + "backrefs": deser_backrefs, + "2026": deser_2026, + "auto": deser_auto, +} + +_SERIALIZERS = { + "legacy": ser_legacy, + "backrefs": ser_backrefs, + "2026": ser_2026, +} + + +def deserialize( + blob: bytes, + fmt: str = "auto", + *, + max_atom_len: int | None = None, + strict: bool = False, +): + """Deserialize bytes into a LazyNode. + + Formats: "auto" (default), "legacy", "backrefs", "2026". + "auto" handles all three formats by inspecting the magic prefix. + + Keyword-only limits (applied to "2026" and "auto" paths): + max_atom_len: largest single atom in bytes (default 1 MiB) + strict: reject overlong varint encodings + + Total input bytes is bounded by the input slice itself; consensus-aware + callers (e.g. chia_rs) should pass their own ``max_atom_len``. + """ + fn = _DESERIALIZERS.get(fmt) + if fn is None: + raise ValueError(f"unknown deserialize format {fmt!r}, expected one of {list(_DESERIALIZERS)}") + if fmt in ("2026", "auto"): + kwargs: dict = {"strict": strict} + if max_atom_len is not None: + kwargs["max_atom_len"] = max_atom_len + return fn(blob, **kwargs) + return fn(blob) + + +def serialize(node, fmt: str = "2026", *, level: int = 0) -> bytes: + """Serialize a LazyNode to bytes. + + Formats: "2026" (default), "legacy", "backrefs". + + For "2026" format, ``level`` selects the compression level. Levels above + the highest implemented level saturate to it, so passing a large number + always means "best available compression". Currently only level 0 + (left-first/fast) is implemented. + """ + fn = _SERIALIZERS.get(fmt) + if fn is None: + raise ValueError(f"unknown serialize format {fmt!r}, expected one of {list(_SERIALIZERS)}") + if fmt == "2026": + return fn(node, level=level) + return fn(node) diff --git a/wheel/python/tests/test_serialize.py b/wheel/python/tests/test_serialize.py index 7f2888c9..15448d99 100644 --- a/wheel/python/tests/test_serialize.py +++ b/wheel/python/tests/test_serialize.py @@ -1,7 +1,9 @@ import io import unittest +from clvm_rs.clvm_rs import clvm_tree_to_lazy_node, ser_2026 from clvm_rs.program import Program +from clvm_rs.serde import deserialize, serialize from clvm_rs.ser import atom_to_byte_iterator @@ -138,7 +140,7 @@ def test_repr_clvm_tree(self): with self.assertRaises(ValueError): Program.fromhex("ff8085") - o = Program.fromhex("ff808185") + o, _ = Program.from_bytes_with_cursor(bytes.fromhex("ff808185"), 0) self.assertEqual(repr(o._unwrapped_pair[0]), "") self.assertEqual(repr(o._unwrapped_pair[1]), "") @@ -157,3 +159,269 @@ def test_large_atom(self): def test_too_large_atom(self): self.assertRaises(ValueError, lambda: Program.fromhex("fc")) self.assertRaises(ValueError, lambda: Program.fromhex("fc8000000000")) + + def test_2026_magic_prefix_and_from_bytes(self): + p = Program.to((1, (2, 3))) + prefixed = p.to_bytes_2026() + self.assertTrue(prefixed.startswith(bytes.fromhex("fdff32303236"))) + p2 = Program.from_bytes(prefixed) + self.assertEqual(p, p2) + + def test_2026_magic_prefix_explicit_deserializer(self): + p = Program.to([1, 2, 3, 4]) + prefixed = serialize(deserialize(bytes(p), "legacy"), "2026") + p2 = Program.from_bytes_2026(prefixed) + self.assertEqual(p, p2) + + def test_backrefs_parser_rejects_2026(self): + p = Program.to((b"a", b"b")) + prefixed = p.to_bytes_2026() + with self.assertRaises(ValueError): + Program.from_bytes_backrefs(prefixed) + + def test_from_bytes_backrefs_with_actual_backrefs(self): + """Test that from_bytes_backrefs actually parses backrefs format (0xfe opcodes).""" + # Construct a tree with shared subtrees that will trigger backref encoding + shared = Program.to([b"shared_atom", b"another_shared_atom"]) + p = Program.to([shared, shared, shared, shared, shared]) + + # Serialize using backrefs format + blob = serialize(deserialize(bytes(p), "legacy"), "backrefs") + + # Assert the blob contains at least one 0xfe backref opcode + self.assertIn(0xfe, blob, "backrefs-encoded blob should contain at least one 0xfe byte") + + # Round-trip through from_bytes_backrefs + p2 = Program.from_bytes_backrefs(blob) + + # Verify equality (by tree hash) + self.assertEqual(p, p2) + + +class ClvmTreeToLazyNodeTest(unittest.TestCase): + """Tests for clvm_tree_to_lazy_node.""" + + def test_basic_roundtrip(self): + """Python tree -> clvm_tree_to_lazy_node -> ser -> deser -> assert equal.""" + for tree in [b"hello", 42, [1, 2, 3], (1, (2, 3)), [], b""]: + p = Program.to(tree) + blob = p.to_bytes_2026() + p2 = Program.from_bytes(blob) + self.assertEqual(p, p2, f"roundtrip failed for {tree!r}") + + def test_shared_subtrees_via_identity(self): + """Shared Python objects should not cause exponential blowup.""" + t = Program.to(b"") + for _ in range(100): + t = Program.to((t, t)) + lazy = clvm_tree_to_lazy_node(t) + self.assertIsNotNone(lazy) + + def test_content_dedup(self): + """Two distinct Python atoms with same bytes produce one atom.""" + a = Program.to(b"same") + b = Program.to(b"same") + self.assertIsNot(a, b) + tree = Program.to((a, b)) + lazy = clvm_tree_to_lazy_node(tree) + left = lazy.pair[0] + right = lazy.pair[1] + self.assertEqual(left.atom, right.atom) + + def test_pair_dedup(self): + """Two distinct Python pairs with same structure share one pair in the allocator.""" + inner1 = Program.to((1, 2)) + inner2 = Program.to((1, 2)) + self.assertIsNot(inner1, inner2) + tree = Program.to((inner1, inner2)) + lazy = clvm_tree_to_lazy_node(tree) + self.assertIsNotNone(lazy.pair) + + def test_equivalence_with_old_roundtrip(self): + """New clvm_tree_to_lazy_node path produces same output as old deser_backrefs path.""" + from clvm_rs.clvm_rs import deser_backrefs + from clvm_rs.ser import sexp_to_bytes + + test_cases = [ + b"hello", + 42, + [1, 2, 3], + (1, (2, 3)), + [], + b"", + [b"a", [b"b", b"c"], b"d"], + (100, (b"text", (30, (50, (90, (b"ab", b"abab")))))), + ] + for tree in test_cases: + p = Program.to(tree) + new_path = ser_2026(clvm_tree_to_lazy_node(p), level=0) + old_path = ser_2026(deser_backrefs(sexp_to_bytes(p)), level=0) + self.assertEqual(new_path, old_path, f"mismatch for {tree!r}") + + def test_deep_tree(self): + """Deeply nested tree should work without stack overflow.""" + p = Program.to(b"leaf") + for _ in range(1000): + p = Program.to((p, b"x")) + lazy = clvm_tree_to_lazy_node(p) + self.assertIsNotNone(lazy) + + def test_nil_atom(self): + """Empty atom (nil) converts correctly.""" + p = Program.to(b"") + lazy = clvm_tree_to_lazy_node(p) + self.assertEqual(lazy.atom, b"") + self.assertIsNone(lazy.pair) + + def test_preserves_tree_structure(self): + """Converted tree has the same structure as the original.""" + p = Program.to((b"a", (b"b", b"c"))) + lazy = clvm_tree_to_lazy_node(p) + self.assertEqual(lazy.pair[0].atom, b"a") + self.assertEqual(lazy.pair[1].pair[0].atom, b"b") + self.assertEqual(lazy.pair[1].pair[1].atom, b"c") + + def test_large_atom(self): + """Large atoms convert without error.""" + big = b"\xab" * 100_000 + p = Program.to(big) + lazy = clvm_tree_to_lazy_node(p) + self.assertEqual(lazy.atom, big) + + +class Serde2026RoundTripTest(unittest.TestCase): + """Comprehensive serde_2026 round-trip tests.""" + + def check_2026_roundtrip(self, tree, levels=(0,)): + """Serialize to serde_2026, deserialize, check equality.""" + p = Program.to(tree) + for level in levels: + blob = ser_2026(clvm_tree_to_lazy_node(p), level=level) + p2 = Program.from_bytes(blob) + self.assertEqual(p, p2, f"roundtrip failed for {tree!r} at level={level}") + + def test_atoms(self): + for atom in [b"", b"\x00", b"\xff", b"hello world", b"\x01" * 1000]: + self.check_2026_roundtrip(atom) + + def test_integers(self): + for n in [0, 1, -1, 127, 128, 255, 256, 65535, -32768, 2**32]: + self.check_2026_roundtrip(n) + + def test_lists(self): + self.check_2026_roundtrip([]) + self.check_2026_roundtrip([1]) + self.check_2026_roundtrip([1, 2, 3, 4, 5]) + self.check_2026_roundtrip([[1, 2], [3, 4], [5, 6]]) + + def test_nested_pairs(self): + self.check_2026_roundtrip((1, 2)) + self.check_2026_roundtrip((1, (2, (3, (4, 5))))) + self.check_2026_roundtrip(((((1, 2), 3), 4), 5)) + + def test_shared_subtrees_serialization(self): + """Shared subtrees should serialize compactly and round-trip.""" + shared = Program.to([1, 2, 3]) + tree = Program.to((shared, (shared, shared))) + blob = tree.to_bytes_2026() + p2 = Program.from_bytes(blob) + self.assertEqual(tree, p2) + + def test_exponential_sharing(self): + """Exponential sharing (2^50 logical nodes) should serialize compactly.""" + t = Program.to(b"x") + for _ in range(50): + t = Program.to((t, t)) + blob = t.to_bytes_2026() + self.assertLess(len(blob), 500) + # Verify it deserializes without error (skip equality check — + # LazyNode.pair creates new wrappers, making tree_hash O(2^N)) + p2 = Program.from_bytes(blob) + self.assertIsNotNone(p2) + + def test_exponential_sharing_small(self): + """Smaller exponential tree (2^15) — full round-trip with equality.""" + t = Program.to(b"y") + for _ in range(15): + t = Program.to((t, t)) + blob = t.to_bytes_2026() + p2 = Program.from_bytes(blob) + self.assertEqual(t, p2) + + def test_repeated_atoms(self): + """Many copies of the same atom should be deduplicated.""" + tree = Program.to([b"dup"] * 100) + blob = tree.to_bytes_2026() + p2 = Program.from_bytes(blob) + self.assertEqual(tree, p2) + + def test_many_distinct_atoms(self): + """Many distinct atoms should all round-trip.""" + tree = Program.to([bytes([i]) for i in range(256)]) + blob = tree.to_bytes_2026() + p2 = Program.from_bytes(blob) + self.assertEqual(tree, p2) + + def test_2026_level_saturates_to_highest_implemented(self): + """`level` saturates: anything above the top implemented level + produces the same bytes as that level. Today only level 0 exists, + so every non-zero level must produce identical output to level 0.""" + shared = Program.to([1, 2, 3]) + tree = Program.to([shared, shared, shared, shared]) + lazy = clvm_tree_to_lazy_node(tree) + blob_0 = ser_2026(lazy, level=0) + for level in (1, 7, 1 << 20, (1 << 32) - 1): + self.assertEqual( + ser_2026(lazy, level=level), + blob_0, + f"level={level} should saturate to level=0 today", + ) + self.assertEqual(Program.from_bytes(blob_0), tree) + + def test_mixed_tree_with_large_atoms(self): + """Mix of large atoms, small atoms, and nested structure.""" + big = b"A" * 10000 + tree = Program.to((big, [1, 2, (big, b"small")])) + self.check_2026_roundtrip(tree) + + def test_deserialize_api_formats(self): + """Test the serde.py deserialize() with different format strings.""" + p = Program.to([42, 99]) + legacy_blob = bytes(p) + node = deserialize(legacy_blob, "legacy") + self.assertEqual(Program.wrap(node), p) + + node = deserialize(legacy_blob, "auto") + self.assertEqual(Program.wrap(node), p) + + prefixed = p.to_bytes_2026() + node = deserialize(prefixed, "auto") + self.assertEqual(Program.wrap(node), p) + + def test_serialize_api_formats(self): + """Test the serde.py serialize() with different format strings.""" + from clvm_rs.clvm_rs import deser_backrefs + p = Program.to([1, 2, 3]) + node = deser_backrefs(bytes(p)) + + legacy_blob = serialize(node, "legacy") + self.assertEqual(Program.from_bytes(legacy_blob), p) + + s2026_blob = serialize(node, "2026") + self.assertTrue(s2026_blob.startswith(bytes.fromhex("fdff32303236"))) + self.assertEqual(Program.from_bytes(s2026_blob), p) + + def test_ser_2026_deser_2026_symmetry(self): + """ser_2026 emits the magic prefix, deser_2026 must accept it.""" + from clvm_rs.clvm_rs import deser_2026, ser_2026 + + p = Program.to([1, 2, (b"shared", b"shared"), 3]) + blob = ser_2026(clvm_tree_to_lazy_node(p)) + self.assertTrue(blob.startswith(bytes.fromhex("fdff32303236"))) + + node = deser_2026(blob) + self.assertEqual(Program.wrap(node), p) + + # Without the magic prefix it must be rejected. + with self.assertRaises(ValueError): + deser_2026(blob[6:]) diff --git a/wheel/src/api.rs b/wheel/src/api.rs index 0494cf6f..0111d24b 100644 --- a/wheel/src/api.rs +++ b/wheel/src/api.rs @@ -1,16 +1,29 @@ #![allow(clippy::useless_conversion)] +use std::collections::HashMap; use std::io; +use std::rc::Rc; use super::lazy_node::LazyNode; use crate::adapt_response::adapt_response; -use clvmr::allocator::Allocator; +use clvmr::allocator::{Allocator, NodePtr}; use clvmr::chia_dialect::ChiaDialect; use clvmr::chia_dialect::{ClvmFlags, MEMPOOL_MODE}; use clvmr::cost::Cost; use clvmr::error::EvalErr; use clvmr::reduction::Response; use clvmr::run_program::run_program; -use clvmr::serde::{ParsedTriple, node_from_bytes, parse_triples, serialized_length_from_bytes}; +use clvmr::serde::{ + ParsedTriple, node_from_bytes, node_from_bytes_backrefs, node_to_bytes, node_to_bytes_backrefs, + parse_triples, serialized_length_from_bytes, +}; +use clvmr::serde_2026::{ + SERDE_2026_MAGIC_PREFIX, deserialize_2026_body, node_from_bytes_serde_2026, + node_to_bytes_serde_2026_level, +}; + +/// Sane "don't OOM the parser" default. clvm_rs has no consensus opinion; +/// downstream wrappers (e.g. chia_rs) supply their own caps. +const PY_DEFAULT_MAX_ATOM_LEN: usize = 1 << 20; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyTuple}; @@ -86,11 +99,221 @@ fn deserialize_as_tree( Ok((r, s)) } +// --- Deserialize functions: bytes -> LazyNode --- + +#[pyfunction] +fn deser_legacy(blob: &[u8]) -> PyResult { + let mut a = Allocator::new(); + let node = node_from_bytes(&mut a, blob).map_err(eval_to_py)?; + Ok(LazyNode::new(Rc::new(a), node)) +} + +#[pyfunction] +fn deser_backrefs(blob: &[u8]) -> PyResult { + let mut a = Allocator::new(); + let node = node_from_bytes_backrefs(&mut a, blob).map_err(eval_to_py)?; + Ok(LazyNode::new(Rc::new(a), node)) +} + +/// Deserialize a serde_2026 blob. The input must start with +/// `SERDE_2026_MAGIC_PREFIX` (the same prefix `ser_2026` emits); the prefix is +/// stripped before calling the underlying decoder. This makes `ser_2026` and +/// `deser_2026` a symmetric pair. +#[pyfunction] +#[pyo3(signature = (blob, *, max_atom_len=PY_DEFAULT_MAX_ATOM_LEN, strict=false))] +fn deser_2026(blob: &[u8], max_atom_len: usize, strict: bool) -> PyResult { + let mut a = Allocator::new(); + let node = node_from_bytes_serde_2026(&mut a, blob, max_atom_len, strict).map_err(|e| { + // Translate the prefix-missing error into a friendlier ValueError. + if !blob.starts_with(SERDE_2026_MAGIC_PREFIX.as_slice()) { + pyo3::exceptions::PyValueError::new_err( + "deser_2026: blob is missing the serde_2026 magic prefix", + ) + } else { + eval_to_py(e) + } + })?; + Ok(LazyNode::new(Rc::new(a), node)) +} + +/// Deserialize CLVM bytes, auto-detecting the format (classic, backrefs, or +/// serde_2026). If the blob starts with the magic prefix +/// `fd ff 32 30 32 36`, it is treated as serde_2026; otherwise the backrefs +/// deserializer is used (which also handles plain classic format). +/// +/// This is a Python convenience function — clvm_rs's Rust API doesn't have +/// an auto-switching counterpart. Consensus-aware callers should sniff the +/// prefix themselves and use their own caps. +#[pyfunction] +#[pyo3(signature = (blob, *, max_atom_len=PY_DEFAULT_MAX_ATOM_LEN, strict=false))] +fn deser_auto(blob: &[u8], max_atom_len: usize, strict: bool) -> PyResult { + let mut a = Allocator::new(); + let node = if let Some(body) = blob.strip_prefix(SERDE_2026_MAGIC_PREFIX.as_slice()) { + deserialize_2026_body(&mut a, body, max_atom_len, strict).map_err(eval_to_py)? + } else { + node_from_bytes_backrefs(&mut a, blob).map_err(eval_to_py)? + }; + Ok(LazyNode::new(Rc::new(a), node)) +} + +// --- Serialize functions: LazyNode -> bytes --- + +#[pyfunction] +fn ser_legacy(py: Python, node: &LazyNode) -> PyResult> { + let bytes = node_to_bytes(node.allocator(), node.node()).map_err(eval_to_py)?; + Ok(PyBytes::new(py, &bytes).unbind()) +} + +#[pyfunction] +fn ser_backrefs(py: Python, node: &LazyNode) -> PyResult> { + let bytes = node_to_bytes_backrefs(node.allocator(), node.node()).map_err(eval_to_py)?; + Ok(PyBytes::new(py, &bytes).unbind()) +} + +/// Serialize to serde_2026 format (always includes the magic prefix). +/// +/// `level` selects the compression level. Levels above the highest implemented +/// level saturate to it, so passing `u32::MAX` always means "best available +/// compression". Currently only level 0 (left-first/fast) is implemented. +#[pyfunction] +#[pyo3(signature = (node, *, level=0))] +fn ser_2026(py: Python, node: &LazyNode, level: u32) -> PyResult> { + let buf = + node_to_bytes_serde_2026_level(node.allocator(), node.node(), level).map_err(eval_to_py)?; + Ok(PyBytes::new(py, &buf).unbind()) +} + +/// Convert a Python CLVM tree (any object with `.atom` / `.pair` attributes) +/// into a `LazyNode` backed by a Rust `Allocator`, with full interning. +/// +/// Uses three hash maps mirroring `intern_tree`: +/// 1. Python object identity (`id()`) -> NodePtr (prevents exponential blowup) +/// 2. Atom byte content -> NodePtr (deduplicates identical atoms) +/// 3. (left, right) pair -> NodePtr (deduplicates structurally identical pairs) +#[pyfunction] +fn clvm_tree_to_lazy_node(obj: Bound<'_, PyAny>) -> PyResult { + let mut allocator = Allocator::new(); + + let mut identity_map: HashMap = HashMap::new(); + let mut atom_map: HashMap, NodePtr> = HashMap::new(); + let mut pair_map: HashMap<(NodePtr, NodePtr), NodePtr> = HashMap::new(); + + enum WorkItem<'py> { + Visit(Bound<'py, PyAny>), + BuildPair { + id: usize, + left_id: usize, + right_id: usize, + }, + } + + let root_ptr = obj.as_ptr() as usize; + let mut stack: Vec> = vec![WorkItem::Visit(obj)]; + + while let Some(item) = stack.pop() { + match item { + WorkItem::Visit(pyobj) => { + let id = pyobj.as_ptr() as usize; + + if identity_map.contains_key(&id) { + continue; + } + + let atom_val: Option> = pyobj.getattr("atom")?.extract()?; + + if let Some(bytes) = atom_val { + let node = if let Some(&existing) = atom_map.get(&bytes) { + existing + } else { + let new_node = allocator + .new_atom(&bytes) + .map_err(|e| pyo3::exceptions::PyMemoryError::new_err(e.to_string()))?; + atom_map.insert(bytes, new_node); + new_node + }; + identity_map.insert(id, node); + } else { + let pair_val: Option<(Bound<'_, PyAny>, Bound<'_, PyAny>)> = + pyobj.getattr("pair")?.extract()?; + + if let Some((left, right)) = pair_val { + let left_id = left.as_ptr() as usize; + let right_id = right.as_ptr() as usize; + + let left_done = identity_map.contains_key(&left_id); + let right_done = identity_map.contains_key(&right_id); + + if left_done && right_done { + let l = identity_map[&left_id]; + let r = identity_map[&right_id]; + let node = if let Some(&existing) = pair_map.get(&(l, r)) { + existing + } else { + let new_node = allocator.new_pair(l, r).map_err(|e| { + pyo3::exceptions::PyMemoryError::new_err(e.to_string()) + })?; + pair_map.insert((l, r), new_node); + new_node + }; + identity_map.insert(id, node); + } else { + stack.push(WorkItem::BuildPair { + id, + left_id, + right_id, + }); + if !right_done { + stack.push(WorkItem::Visit(right)); + } + if !left_done { + stack.push(WorkItem::Visit(left)); + } + } + } else { + return Err(pyo3::exceptions::PyValueError::new_err( + "CLVM object has neither .atom nor .pair", + )); + } + } + } + WorkItem::BuildPair { + id, + left_id, + right_id, + } => { + let l = identity_map[&left_id]; + let r = identity_map[&right_id]; + let node = if let Some(&existing) = pair_map.get(&(l, r)) { + existing + } else { + let new_node = allocator + .new_pair(l, r) + .map_err(|e| pyo3::exceptions::PyMemoryError::new_err(e.to_string()))?; + pair_map.insert((l, r), new_node); + new_node + }; + identity_map.insert(id, node); + } + } + } + + let root = identity_map[&root_ptr]; + Ok(LazyNode::new(Rc::new(allocator), root)) +} + #[pymodule] fn clvm_rs(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(run_serialized_chia_program, m)?)?; m.add_function(wrap_pyfunction!(serialized_length, m)?)?; m.add_function(wrap_pyfunction!(deserialize_as_tree, m)?)?; + m.add_function(wrap_pyfunction!(deser_legacy, m)?)?; + m.add_function(wrap_pyfunction!(deser_backrefs, m)?)?; + m.add_function(wrap_pyfunction!(deser_2026, m)?)?; + m.add_function(wrap_pyfunction!(deser_auto, m)?)?; + m.add_function(wrap_pyfunction!(ser_legacy, m)?)?; + m.add_function(wrap_pyfunction!(ser_backrefs, m)?)?; + m.add_function(wrap_pyfunction!(ser_2026, m)?)?; + m.add_function(wrap_pyfunction!(clvm_tree_to_lazy_node, m)?)?; m.add("NO_UNKNOWN_OPS", ClvmFlags::NO_UNKNOWN_OPS.bits())?; m.add("LIMIT_HEAP", ClvmFlags::LIMIT_HEAP.bits())?; diff --git a/wheel/src/lazy_node.rs b/wheel/src/lazy_node.rs index 37894257..e103cced 100644 --- a/wheel/src/lazy_node.rs +++ b/wheel/src/lazy_node.rs @@ -46,4 +46,14 @@ impl LazyNode { node: n, } } + + // Rust-side serializers need direct access to the backing allocator/node. + // These are intentionally crate-local; Python only sees the atom/pair view. + pub fn allocator(&self) -> &Allocator { + &self.allocator + } + + pub fn node(&self) -> NodePtr { + self.node + } }