From fff1bed3173f75866626a6f5f85218232b6d65ed Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 29 Jul 2026 15:09:08 -0700 Subject: [PATCH] Accept bare serde_2026 generator blobs at the load boundary A serde_2026 blob that does not start with the 6-byte magic prefix gets the prefix prepended before deserialization. Readers accept both framings; the deserializer, the builder emit path and the wire format stay strict. This lets a future database format drop the 6 bytes per block without a flag day, and without ever circulating bare blobs (which old parsers silently misparse) inside the node. Co-authored-by: Cursor --- .../chia-consensus/src/run_block_generator.rs | 92 +++++++++-- crates/chia-consensus/src/serde_2026.rs | 155 +++++++++++++++++- 2 files changed, 229 insertions(+), 18 deletions(-) diff --git a/crates/chia-consensus/src/run_block_generator.rs b/crates/chia-consensus/src/run_block_generator.rs index 490d90df3..d7ca0b619 100644 --- a/crates/chia-consensus/src/run_block_generator.rs +++ b/crates/chia-consensus/src/run_block_generator.rs @@ -831,6 +831,8 @@ mod tests { #[test] fn test_old_serialization_rejected_with_interned_flag() { + use clvmr::serde::node_to_bytes_backrefs; + // with INTERNED_GENERATOR active, an otherwise-valid generator in the // old (classic/backrefs) serialization is a consensus failure let program = make_generator(1); @@ -849,19 +851,83 @@ mod tests { ); assert!(result.is_ok(), "sanity: valid without INTERNED_GENERATOR"); + // the same generator in the backrefs encoding (also starts with the + // 0xff pair marker) + let mut a = Allocator::new(); + let node = node_from_bytes(&mut a, &program).expect("node_from_bytes"); + let backrefs = node_to_bytes_backrefs(&a, node).expect("node_to_bytes_backrefs"); + assert!(backrefs.starts_with(&[0xff, 0x01])); + + // and the classic blob wrapped in the serde_2026 magic prefix (so it + // is passed through, not prepended to — both framings of the + // prefix check must reject old encodings) + let mut wrapped = SERDE_2026_MAGIC_PREFIX.to_vec(); + wrapped.extend_from_slice(&program); + let flags = flags | ConsensusFlags::INTERNED_GENERATOR; - let result = run_block_generator2( - &program, - blocks, - u64::MAX, - flags, - &Signature::default(), - None, - &TEST_CONSTANTS, - ); - assert_eq!( - result.unwrap_err(), - ValidationErr::Eval(clvmr::error::EvalErr::SerializationError), - ); + for blob in [program.as_slice(), &backrefs, &wrapped] { + // after normalization the blob is a garbage serde_2026 body: its + // leading 0xff is an invalid varint first byte + let result = run_block_generator2( + blob, + blocks, + u64::MAX, + flags, + &Signature::default(), + None, + &TEST_CONSTANTS, + ); + assert_eq!( + result.unwrap_err(), + ValidationErr::Eval(clvmr::error::EvalErr::SerializationError), + ); + } + } + + #[test] + fn test_bare_serde_2026_body_accepted_with_interned_flag() { + use crate::solution_generator::solution_generator_2026; + + // generous parsing: a serde_2026 generator stripped of its magic + // prefix (e.g. loaded from a future database format that saves the + // 6 bytes) validates identically to the prefixed wire form + let flags = ConsensusFlags::DONT_VALIDATE_SIGNATURE + | ConsensusFlags::SIMPLE_GENERATOR + | ConsensusFlags::INTERNED_GENERATOR; + let blocks: &[&[u8]] = &[]; + + let puzzle_hash = tree_hash_atom(&[1]).to_bytes(); + let empty_solution: &[u8] = &[0x80]; + let spends = [( + Coin::new([0u8; 32].into(), puzzle_hash.into(), 0), + IDENTITY_PUZZLE, + empty_solution, + )]; + let program = solution_generator_2026(spends).expect("solution_generator_2026"); + assert!(program.starts_with(&SERDE_2026_MAGIC_PREFIX)); + let body = &program[SERDE_2026_MAGIC_PREFIX.len()..]; + + let run = |blob: &[u8]| { + let (_, conds) = run_block_generator2( + blob, + blocks, + u64::MAX, + flags, + &Signature::default(), + None, + &TEST_CONSTANTS, + ) + .expect("run_block_generator2"); + conds + }; + let prefixed = run(&program); + let bare = run(body); + assert_eq!(prefixed.spends.len(), 1); + assert_eq!(bare.spends.len(), 1); + assert_eq!(prefixed.spends[0].coin_id, bare.spends[0].coin_id); + // cost is charged on the interned tree, not wire bytes, so both + // framings cost the same + assert_eq!(prefixed.cost, bare.cost); + assert_eq!(prefixed.execution_cost, bare.execution_cost); } } diff --git a/crates/chia-consensus/src/serde_2026.rs b/crates/chia-consensus/src/serde_2026.rs index c2056f98e..31460d154 100644 --- a/crates/chia-consensus/src/serde_2026.rs +++ b/crates/chia-consensus/src/serde_2026.rs @@ -6,24 +6,67 @@ //! "sniff the magic prefix and dispatch" convenience that callers used to get //! from `clvm_rs::serde::node_from_bytes_auto`. +use std::borrow::Cow; + use clvmr::allocator::{Allocator, NodePtr}; use clvmr::error::{EvalErr, Result}; use clvmr::serde::{SERDE_2026_MAGIC_PREFIX, deserialize_2026, node_from_bytes_backrefs}; +/// Normalize a serde_2026 blob to its canonical, magic-prefixed form. +/// +/// If `blob` already starts with [`SERDE_2026_MAGIC_PREFIX`], it is passed +/// through unchanged (no copy); otherwise a copy with the prefix prepended +/// is returned. +/// +/// The direction matters: readers normalize by *prepending*, never by +/// stripping. The prefixed form is the only in-memory / interchange form, so +/// a blob that leaks out of the node (an RPC response, a cache dump, a .bin +/// file) always carries the prefix and fails fast in old deserializers — +/// the leading `0xfd` declares an atom length far beyond their hard cap. A +/// *bare* body, by contrast, "successfully" misparses as a small garbage +/// atom in old software, so bare bodies may exist only at rest (e.g. a +/// database format that saves the 6 bytes); this helper re-arms them at the +/// load boundary. +/// +/// The prepend-or-passthrough decision is structurally unambiguous: no +/// valid serde_2026 body can begin with `0xfd`. A varint whose first byte is +/// `0xfd` has six leading ones and its single first-byte payload bit set — +/// that bit is the sign bit of the two's-complement value, so the varint +/// always decodes negative, and the body's leading varint (the atom-group +/// count) must be non-negative. This holds for overlong (non-strict) +/// encodings too, so a bare body can never be mistaken for a prefixed blob. +pub fn ensure_serde_2026_prefix(blob: &[u8]) -> Cow<'_, [u8]> { + if blob.starts_with(&SERDE_2026_MAGIC_PREFIX) { + Cow::Borrowed(blob) + } else { + let mut prefixed = Vec::with_capacity(SERDE_2026_MAGIC_PREFIX.len() + blob.len()); + prefixed.extend_from_slice(&SERDE_2026_MAGIC_PREFIX); + prefixed.extend_from_slice(blob); + Cow::Owned(prefixed) + } +} + /// Deserialize a generator on the consensus path: the blob must be a -/// magic-prefixed serde_2026 encoding, with no fallback to classic/backrefs -/// parsing — with `INTERNED_GENERATOR` active, serde_2026 is the only legal -/// generator encoding. `max_blob_size` and `strict = false` have the same -/// meaning (and rationale) as in [`node_from_bytes_auto`]. +/// serde_2026 encoding, with no fallback to classic/backrefs parsing — with +/// `INTERNED_GENERATOR` active, serde_2026 is the only legal generator +/// encoding. Both framings are accepted: the magic-prefixed wire form and +/// the bare body (normalized via [`ensure_serde_2026_prefix`], which +/// prepends the prefix — the underlying deserializer still requires it). +/// Classic/backrefs blobs fail either way: their leading byte makes for an +/// invalid serde_2026 body. `max_blob_size` and `strict = false` have the +/// same meaning (and rationale) as in [`node_from_bytes_auto`]. pub fn node_from_bytes_2026( allocator: &mut Allocator, bytes: &[u8], max_blob_size: usize, ) -> Result { + let bytes = ensure_serde_2026_prefix(bytes); + // the size cap applies to the canonical (prefixed) form, so both + // framings of the same generator are accepted or rejected together if bytes.len() > max_blob_size { return Err(EvalErr::SerializationError); } - deserialize_2026(allocator, bytes, max_blob_size, false) + deserialize_2026(allocator, &bytes, max_blob_size, false) } /// Compression level passed to [`clvmr::serde::serialize_2026`] when chia @@ -109,6 +152,12 @@ pub fn max_canonical_blob_size(max_cost: u64, cost_per_byte: u64) -> usize { /// blob of at least `L` bytes — no atom of a cost-valid generator can ever /// exceed the blob bound. There is deliberately no separate atom-length /// constant. +/// +/// Do not pass *bare* (prefix-stripped) serde_2026 bodies here: without the +/// prefix they are indistinguishable from classic encodings and would +/// silently misparse via the backrefs fallback. A blob known to be +/// serde_2026 belongs in [`node_from_bytes_2026`], which accepts both +/// framings. pub fn node_from_bytes_auto( allocator: &mut Allocator, bytes: &[u8], @@ -264,6 +313,102 @@ mod tests { } } + #[test] + fn test_ensure_serde_2026_prefix() { + let mut a = Allocator::new(); + let node = sample_tree(&mut a); + let blob = serialize_2026(&a, node, SERDE_2026_COMPRESSION_LEVEL).unwrap(); + + // prefixed blob: passed through without copying + assert!(matches!(ensure_serde_2026_prefix(&blob), Cow::Borrowed(_))); + + // bare body: normalizing prepends the prefix, recovering the wire form + let body = &blob[SERDE_2026_MAGIC_PREFIX.len()..]; + let normalized = ensure_serde_2026_prefix(body); + assert!(matches!(normalized, Cow::Owned(_))); + assert_eq!(normalized.as_ref(), blob.as_slice()); + } + + #[test] + fn test_node_from_bytes_2026_accepts_both_framings() { + use clvm_utils::tree_hash; + + let mut a = Allocator::new(); + let node = sample_tree(&mut a); + let expected_bytes = node_to_bytes(&a, node).unwrap(); + let expected_hash = tree_hash(&a, node); + let blob = serialize_2026(&a, node, SERDE_2026_COMPRESSION_LEVEL).unwrap(); + let body = &blob[SERDE_2026_MAGIC_PREFIX.len()..]; + + for framing in [blob.as_slice(), body] { + let mut b = Allocator::new(); + let parsed = + node_from_bytes_2026(&mut b, framing, mainnet_cap()).expect("node_from_bytes_2026"); + assert_eq!(node_to_bytes(&b, parsed).unwrap(), expected_bytes); + assert_eq!(tree_hash(&b, parsed), expected_hash); + } + } + + #[test] + fn test_node_from_bytes_2026_body_with_multibyte_group_count() { + // >63 distinct atom lengths force a multi-byte group-count header, + // so the bare body starts with a high varint byte (0x80..=0xbf) — + // still distinct from the 0xfd magic, so it must be prepended to + // and parse like the prefixed form. + let mut a = Allocator::new(); + let mut node = a.nil(); + for len in 1..=64usize { + let atom = a.new_atom(&vec![0x5a; len]).unwrap(); + node = a.new_pair(atom, node).unwrap(); + } + let blob = serialize_2026(&a, node, SERDE_2026_COMPRESSION_LEVEL).unwrap(); + let body = &blob[SERDE_2026_MAGIC_PREFIX.len()..]; + assert!((0x80..=0xbf).contains(&body[0])); + + let mut b = Allocator::new(); + let parsed = + node_from_bytes_2026(&mut b, body, mainnet_cap()).expect("node_from_bytes_2026"); + assert_eq!( + node_to_bytes(&b, parsed).unwrap(), + node_to_bytes(&a, node).unwrap() + ); + } + + #[rstest] + #[case::empty(&[])] + #[case::first_prefix_byte_only(&[0xfd])] + #[case::shorter_than_prefix(&SERDE_2026_MAGIC_PREFIX[..5])] + #[case::exactly_the_prefix(&SERDE_2026_MAGIC_PREFIX)] + // classic/backrefs generators serialize a pair, so they start with 0xff + // — an invalid varint first byte in either framing of the check + #[case::classic_bare(&[0xff, 0x01, 0x80])] + #[case::classic_wrapped_in_prefix(&[0xfd, 0xff, b'2', b'0', b'2', b'6', 0xff, 0x01, 0x80])] + fn test_node_from_bytes_2026_rejects(#[case] blob: &[u8]) { + let mut a = Allocator::new(); + assert!(matches!( + node_from_bytes_2026(&mut a, blob, mainnet_cap()), + Err(EvalErr::SerializationError) + )); + } + + #[test] + fn test_node_from_bytes_2026_cap_applies_to_canonical_form() { + let mut a = Allocator::new(); + let node = sample_tree(&mut a); + let blob = serialize_2026(&a, node, SERDE_2026_COMPRESSION_LEVEL).unwrap(); + let body = &blob[SERDE_2026_MAGIC_PREFIX.len()..]; + + // the bare body is rejected or accepted exactly like its prefixed + // equivalent: the cap counts the prepended prefix + let mut b = Allocator::new(); + assert!(matches!( + node_from_bytes_2026(&mut b, body, blob.len() - 1), + Err(EvalErr::SerializationError) + )); + let mut b = Allocator::new(); + node_from_bytes_2026(&mut b, body, blob.len()).expect("node_from_bytes_2026"); + } + #[test] fn test_auto_dispatch_all_formats() { let mut a = Allocator::new();