more datalayer fuzzers

crates/chia-datalayer/fuzz/Cargo.toml

@@ -31,3 +31,17 @@ path = "fuzz_targets/merkle_blob_insert_and_delete.rs"
 test = false
 doc = false
 bench = false
+
+[[bin]]
+name = "iterate_all_once"
+path = "fuzz_targets/iterate_all_once.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "proofs_of_inclusion"
+path = "fuzz_targets/proofs_of_inclusion.rs"
+test = false
+doc = false
+bench = false

crates/chia-datalayer/fuzz/fuzz_targets/iterate_all_once.rs

@@ -0,0 +1,61 @@
+#![no_main]
+
+use libfuzzer_sys::{
+    arbitrary::{Arbitrary, Unstructured},
+    fuzz_target,
+};
+use std::collections::HashMap;
+
+use chia_datalayer::{
+    Block, BreadthFirstIterator, Error, Hash, InsertLocation, KeyId, LeftChildFirstIterator,
+    MerkleBlob, NodeType, ParentFirstIterator, TreeIndex, ValueId,
+};
+
+fuzz_target!(|data: &[u8]| {
+    let mut blob = MerkleBlob::new(Vec::new()).unwrap();
+    blob.check_integrity_on_drop = false;
+
+    let mut leaf_count: usize = 0;
+
+    let mut unstructured = Unstructured::new(data);
+    while !unstructured.is_empty() {
+        let key = KeyId::arbitrary(&mut unstructured).unwrap();
+        let value = ValueId::arbitrary(&mut unstructured).unwrap();
+        let hash = Hash::arbitrary(&mut unstructured).unwrap();
+
+        match blob.insert(key, value, &hash, InsertLocation::Auto {}) {
+            Ok(_) => {
+                leaf_count += 1;
+            }
+            // should remain valid through these errors
+            Err(Error::KeyAlreadyPresent()) => continue,
+            Err(Error::HashAlreadyPresent()) => continue,
+            // other errors should not be occurring
+            Err(error) => panic!("unexpected error while inserting: {:?}", error),
+        };
+    }
+
+    blob.check_integrity().unwrap();
+
+    let raw_blob = blob.read_blob();
+
+    let nodes_a = LeftChildFirstIterator::new(raw_blob, None)
+        .collect::<Result<HashMap<TreeIndex, Block>, Error>>()
+        .unwrap();
+    let nodes_b = ParentFirstIterator::new(raw_blob, None)
+        .collect::<Result<HashMap<TreeIndex, Block>, Error>>()
+        .unwrap();
+    let nodes_c = BreadthFirstIterator::new(raw_blob, None)
+        .collect::<Result<HashMap<TreeIndex, Block>, Error>>()
+        .unwrap();
+
+    assert_eq!(nodes_c.len(), leaf_count);
+
+    assert_eq!(nodes_a, nodes_b);
+    let nodes_a_leafs: HashMap<TreeIndex, Block> = nodes_a
+        .iter()
+        .filter(|(_index, block)| block.metadata.node_type == NodeType::Leaf)
+        .map(|(&k, &v)| (k, v))
+        .collect();
+    assert_eq!(nodes_a_leafs, nodes_c);
+});

crates/chia-datalayer/fuzz/fuzz_targets/proofs_of_inclusion.rs

@@ -0,0 +1,41 @@
+#![no_main]
+
+use libfuzzer_sys::{
+    arbitrary::{Arbitrary, Unstructured},
+    fuzz_target,
+};
+
+use chia_datalayer::{Error, Hash, InsertLocation, KeyId, MerkleBlob, ValueId};
+
+fuzz_target!(|data: &[u8]| {
+    let mut blob = MerkleBlob::new(Vec::new()).unwrap();
+    blob.check_integrity_on_drop = false;
+
+    let mut keys: Vec<KeyId> = Vec::new();
+
+    let mut unstructured = Unstructured::new(data);
+    while !unstructured.is_empty() {
+        let key = KeyId::arbitrary(&mut unstructured).unwrap();
+        let value = ValueId::arbitrary(&mut unstructured).unwrap();
+        let hash = Hash::arbitrary(&mut unstructured).unwrap();
+
+        match blob.insert(key, value, &hash, InsertLocation::Auto {}) {
+            Ok(_) => {
+                keys.push(key);
+            }
+            // should remain valid through these errors
+            Err(Error::KeyAlreadyPresent()) => continue,
+            Err(Error::HashAlreadyPresent()) => continue,
+            // other errors should not be occurring
+            Err(error) => panic!("unexpected error while inserting: {:?}", error),
+        };
+    }
+
+    blob.calculate_lazy_hashes().unwrap();
+    blob.check_integrity().unwrap();
+
+    for key in keys {
+        let proof = blob.get_proof_of_inclusion(key).unwrap();
+        assert!(proof.valid());
+    }
+});

crates/chia-datalayer/src/merkle/blob.rs

@@ -1071,32 +1071,39 @@ impl MerkleBlob {
         Ok(self.get_block(index)?.node.parent())
     }
 
-    pub fn get_lineage_with_indexes(
+    pub fn get_lineage_blocks_with_indexes(
         &self,
         index: TreeIndex,
-    ) -> Result<Vec<(TreeIndex, Node)>, Error> {
+    ) -> Result<Vec<(TreeIndex, Block)>, Error> {
         let mut next_index = Some(index);
         let mut lineage = vec![];
 
         while let Some(this_index) = next_index {
-            let node = self.get_node(this_index)?;
-            next_index = node.parent().0;
-            lineage.push((this_index, node));
+            let block = self.get_block(this_index)?;
+            next_index = block.node.parent().0;
+            lineage.push((this_index, block));
         }
 
         Ok(lineage)
     }
 
-    pub fn get_lineage_indexes(&self, index: TreeIndex) -> Result<Vec<TreeIndex>, Error> {
-        let mut next_index = Some(index);
-        let mut lineage: Vec<TreeIndex> = vec![];
-
-        while let Some(this_index) = next_index {
-            lineage.push(this_index);
-            next_index = self.get_parent_index(this_index)?.0;
-        }
+    pub fn get_lineage_with_indexes(
+        &self,
+        index: TreeIndex,
+    ) -> Result<Vec<(TreeIndex, Node)>, Error> {
+        Ok(self
+            .get_lineage_blocks_with_indexes(index)?
+            .iter()
+            .map(|(index, block)| (*index, block.node))
+            .collect())
+    }
 
-        Ok(lineage)
+    pub fn get_lineage_indexes(&self, index: TreeIndex) -> Result<Vec<TreeIndex>, Error> {
+        Ok(self
+            .get_lineage_blocks_with_indexes(index)?
+            .iter()
+            .map(|(index, _block)| *index)
+            .collect())
     }
 
     // pub fn iter(&self) -> MerkleBlobLeftChildFirstIterator<'_> {
@@ -1162,13 +1169,18 @@ impl MerkleBlob {
             .get_node(index)?
             .expect_leaf("key to index mapping should only have leaves");
 
-        let parents = self.get_lineage_with_indexes(index)?;
+        let parents = self.get_lineage_blocks_with_indexes(index)?;
         let mut layers: Vec<proof_of_inclusion::ProofOfInclusionLayer> = Vec::new();
         let mut parents_iter = parents.iter();
         // first in the lineage is the index itself, second is the first parent
         parents_iter.next();
-        for (next_index, parent) in parents_iter {
-            let parent = parent.expect_internal("all nodes after the first should be internal");
+        for (next_index, block) in parents_iter {
+            if block.metadata.dirty {
+                return Err(Error::Dirty(*next_index));
+            }
+            let parent = block
+                .node
+                .expect_internal("all nodes after the first should be internal");
             let sibling_index = parent.sibling_index(index)?;
             let sibling_block = self.get_block(sibling_index)?;
             let sibling = sibling_block.node;
@@ -1332,6 +1344,10 @@ impl MerkleBlob {
             }
         }
     }
+
+    pub fn read_blob(&self) -> &Vec<u8> {
+        &self.blob
+    }
 }
 
 #[cfg(feature = "py-bindings")]

crates/chia-datalayer/src/merkle/format.rs

@@ -214,7 +214,7 @@ pub struct LeafNode {
 }
 
 // TODO: consider forcing ::new() with validity checks
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum Node {
     Internal(InternalNode),
     Leaf(LeafNode),
@@ -315,6 +315,7 @@ impl<'py> IntoPyObject<'py> for Node {
 }
 
 // TODO: consider forcing ::new() with validity checks
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub struct Block {
     // NOTE: metadata node type and node's type not verified for agreement
     pub metadata: NodeMetadata,