#75 Switch SMT inclusion verification from version 3o to 6a

src/main/java/org/unicitylabs/sdk/api/InclusionCertificate.java

@@ -5,6 +5,7 @@
 import org.unicitylabs.sdk.crypto.hash.HashAlgorithm;
 import org.unicitylabs.sdk.smt.radix.FinalizedBranch;
 import org.unicitylabs.sdk.smt.radix.FinalizedLeafBranch;
+import org.unicitylabs.sdk.smt.SparseMerkleTreePathUtils;
 import org.unicitylabs.sdk.smt.radix.FinalizedNodeBranch;
 import org.unicitylabs.sdk.util.BitString;
 import org.unicitylabs.sdk.util.HexConverter;
@@ -137,6 +138,7 @@ public boolean verify(StateId leafKey, DataHash leafValue, DataHash expectedRoot
       hash = new DataHasher(HashAlgorithm.SHA256)
               .update(new byte[]{0x01})
               .update(LongConverter.encode(depth))
+              .update(SparseMerkleTreePathUtils.pathToRegion(keyPath, depth))
               .update(left)
               .update(right)
               .digest();

src/main/java/org/unicitylabs/sdk/smt/SparseMerkleTreePathUtils.java

@@ -0,0 +1,47 @@
+package org.unicitylabs.sdk.smt;
+
+import java.math.BigInteger;
+import java.util.Objects;
+
+/**
+ * Path utilities for the radix sparse Merkle trees.
+ */
+public final class SparseMerkleTreePathUtils {
+
+  /**
+   * Region size in bytes: the SMT key length, since the region holds a full 256-bit key prefix. A
+   * fixed protocol constant that must match the JS SDK; it is not a tunable parameter.
+   */
+  private static final int REGION_LENGTH = 32;
+
+  private SparseMerkleTreePathUtils() {
+  }
+
+  /**
+   * Region committed by an interior node: the `depth`-bit common prefix of all leaves in the node's
+   * sub-tree. The `i`th lowest bit of path will be the `i mod 8`th lowest bit in the `i div 8`th
+   * byte of the returned 32-byte array (so the packing is little-endian); the remaining bits of the array
+   * are set to zero.
+   *
+   * @param path absolute node or key path
+   * @param depth bifurcation depth of the node
+   * @return 32-byte region
+   * @throws NullPointerException if {@code path} is {@code null}
+   */
+  public static byte[] pathToRegion(BigInteger path, int depth) {
+    Objects.requireNonNull(path, "path cannot be null");
+
+    byte[] region = new byte[REGION_LENGTH];
+    int fullBytes = depth / 8;
+    int remainderBits = depth % 8;
+
+    BigInteger bits = path;
+    for (int j = 0; j < fullBytes && j < REGION_LENGTH; j++, bits = bits.shiftRight(8)) {
+      region[j] = bits.byteValue();
+    }
+    if (remainderBits > 0 && fullBytes < REGION_LENGTH) {
+      region[fullBytes] = (byte) (bits.byteValue() & ((1 << remainderBits) - 1));
+    }
+    return region;
+  }
+}

src/main/java/org/unicitylabs/sdk/smt/radix/FinalizedNodeBranch.java

@@ -3,6 +3,7 @@
 import org.unicitylabs.sdk.crypto.hash.DataHash;
 import org.unicitylabs.sdk.crypto.hash.DataHasher;
 import org.unicitylabs.sdk.crypto.hash.HashAlgorithm;
+import org.unicitylabs.sdk.smt.SparseMerkleTreePathUtils;
 import org.unicitylabs.sdk.util.LongConverter;
 
 import java.math.BigInteger;
@@ -72,6 +73,7 @@ public static FinalizedNodeBranch fromPendingNode(HashAlgorithm hashAlgorithm, P
     DataHash hash = new DataHasher(hashAlgorithm)
             .update(new byte[]{0x01})
             .update(LongConverter.encode(node.getDepth()))
+            .update(SparseMerkleTreePathUtils.pathToRegion(node.getPath(), node.getDepth()))
             .update(left.getHash().getData())
             .update(right.getHash().getData())
             .digest();

src/main/java/org/unicitylabs/sdk/smt/radix/SparseMerkleTree.java

@@ -59,7 +59,7 @@ public synchronized void addLeaf(byte[] key, byte[] data)
     boolean isRight = path.testBit(0);
     Branch branch = isRight ? this.right : this.left;
     Branch result = branch != null
-            ? SparseMerkleTree.buildTree(branch, path, 0, key, data)
+            ? SparseMerkleTree.buildTree(branch, path, key, data)
             : new PendingLeafBranch(path, key, data);
 
     if (isRight) {
@@ -83,13 +83,12 @@ public synchronized FinalizedNodeBranch calculateRoot() {
     return new PendingNodeBranch(BigInteger.ONE, 0, left, right).finalize(hashAlgorithm);
   }
 
-  private static Branch buildTree(Branch branch, BigInteger remainingPath, int depth, byte[] key,
-                                  byte[] value) throws BranchExistsException, LeafOutOfBoundsException {
-    CommonPath commonPath = CommonPath.create(remainingPath, branch.getPath());
-    int commonPathLength = commonPath.getLength();
-    boolean isRight = remainingPath.shiftRight(commonPathLength).testBit(0);
+  private static Branch buildTree(Branch branch, BigInteger keyPath, byte[] key, byte[] value)
+          throws BranchExistsException, LeafOutOfBoundsException {
+    CommonPath commonPath = CommonPath.create(keyPath, branch.getPath());
+    boolean isRight = keyPath.shiftRight(commonPath.getLength()).testBit(0);
 
-    if (commonPath.getPath().equals(remainingPath)) {
+    if (commonPath.getPath().equals(keyPath)) {
       throw new BranchExistsException();
     }
 
@@ -98,40 +97,28 @@ private static Branch buildTree(Branch branch, BigInteger remainingPath, int dep
         throw new LeafOutOfBoundsException();
       }
 
-      LeafBranch leafBranch = (LeafBranch) branch;
-
-      LeafBranch oldBranch = new PendingLeafBranch(
-              branch.getPath().shiftRight(commonPathLength), leafBranch.getKey(),
-              leafBranch.getValue());
-      LeafBranch newBranch = new PendingLeafBranch(
-              remainingPath.shiftRight(commonPathLength), key, value);
-      return new PendingNodeBranch(commonPath.getPath(), depth + commonPathLength,
-              isRight ? oldBranch : newBranch, isRight ? newBranch : oldBranch);
+      LeafBranch newBranch = new PendingLeafBranch(keyPath, key, value);
+      return new PendingNodeBranch(commonPath.getPath(), commonPath.getLength(),
+              isRight ? branch : newBranch, isRight ? newBranch : branch);
     }
 
     NodeBranch nodeBranch = (NodeBranch) branch;
 
     // if node branch is split in the middle
-    if (commonPath.getPath().compareTo(branch.getPath()) < 0) {
-      LeafBranch newBranch = new PendingLeafBranch(
-              remainingPath.shiftRight(commonPathLength), key, value);
-      NodeBranch oldBranch = new PendingNodeBranch(
-              branch.getPath().shiftRight(commonPathLength), nodeBranch.getDepth(),
-              nodeBranch.getLeft(), nodeBranch.getRight());
-      return new PendingNodeBranch(commonPath.getPath(), depth + commonPathLength,
-              isRight ? oldBranch : newBranch, isRight ? newBranch : oldBranch);
+    if (!commonPath.getPath().equals(branch.getPath())) {
+      LeafBranch newBranch = new PendingLeafBranch(keyPath, key, value);
+      return new PendingNodeBranch(commonPath.getPath(), commonPath.getLength(),
+              isRight ? branch : newBranch, isRight ? newBranch : branch);
     }
 
     if (isRight) {
       return new PendingNodeBranch(nodeBranch.getPath(), nodeBranch.getDepth(),
               nodeBranch.getLeft(),
-              SparseMerkleTree.buildTree(nodeBranch.getRight(),
-                      remainingPath.shiftRight(commonPathLength), depth + commonPathLength, key, value));
+              SparseMerkleTree.buildTree(nodeBranch.getRight(), keyPath, key, value));
     }
 
     return new PendingNodeBranch(nodeBranch.getPath(), nodeBranch.getDepth(),
-            SparseMerkleTree.buildTree(nodeBranch.getLeft(),
-                    remainingPath.shiftRight(commonPathLength), depth + commonPathLength, key, value),
+            SparseMerkleTree.buildTree(nodeBranch.getLeft(), keyPath, key, value),
             nodeBranch.getRight());
   }
 }

src/main/java/org/unicitylabs/sdk/smt/radixsum/SparseMerkleSumTree.java

@@ -67,7 +67,7 @@ public synchronized void addLeaf(byte[] key, byte[] data, BigInteger value)
     boolean isRight = path.testBit(0);
     Branch branch = isRight ? this.right : this.left;
     Branch result = branch != null
-            ? SparseMerkleSumTree.buildTree(branch, path, 0, key, data, value)
+            ? SparseMerkleSumTree.buildTree(branch, path, key, data, value)
             : new PendingLeafBranch(path, key, data, value);
 
     if (isRight) {
@@ -91,14 +91,13 @@ public synchronized SparseMerkleSumTreeRootNode calculateRoot() {
     return SparseMerkleSumTreeRootNode.create(left, right, this.hashAlgorithm);
   }
 
-  private static Branch buildTree(Branch branch, BigInteger remainingPath, int depth, byte[] key,
-                                  byte[] data, BigInteger value)
+  private static Branch buildTree(Branch branch, BigInteger keyPath, byte[] key, byte[] data,
+                                  BigInteger value)
           throws BranchExistsException, LeafOutOfBoundsException {
-    CommonPath commonPath = CommonPath.create(remainingPath, branch.getPath());
-    int commonPathLength = commonPath.getLength();
-    boolean isRight = remainingPath.shiftRight(commonPathLength).testBit(0);
+    CommonPath commonPath = CommonPath.create(keyPath, branch.getPath());
+    boolean isRight = keyPath.shiftRight(commonPath.getLength()).testBit(0);
 
-    if (commonPath.getPath().equals(remainingPath)) {
+    if (commonPath.getPath().equals(keyPath)) {
       throw new BranchExistsException();
     }
 
@@ -107,42 +106,28 @@ private static Branch buildTree(Branch branch, BigInteger remainingPath, int dep
         throw new LeafOutOfBoundsException();
       }
 
-      LeafBranch leafBranch = (LeafBranch) branch;
-
-      LeafBranch oldBranch = new PendingLeafBranch(
-              branch.getPath().shiftRight(commonPathLength), leafBranch.getKey(),
-              leafBranch.getData(), leafBranch.getValue());
-      LeafBranch newBranch = new PendingLeafBranch(
-              remainingPath.shiftRight(commonPathLength), key, data, value);
-      return new PendingNodeBranch(commonPath.getPath(), depth + commonPathLength,
-              isRight ? oldBranch : newBranch, isRight ? newBranch : oldBranch);
+      LeafBranch newBranch = new PendingLeafBranch(keyPath, key, data, value);
+      return new PendingNodeBranch(commonPath.getPath(), commonPath.getLength(),
+              isRight ? branch : newBranch, isRight ? newBranch : branch);
     }
 
     NodeBranch nodeBranch = (NodeBranch) branch;
 
     // if node branch is split in the middle
-    if (commonPath.getPath().compareTo(branch.getPath()) < 0) {
-      LeafBranch newBranch = new PendingLeafBranch(
-              remainingPath.shiftRight(commonPathLength), key, data, value);
-      NodeBranch oldBranch = new PendingNodeBranch(
-              branch.getPath().shiftRight(commonPathLength), nodeBranch.getDepth(),
-              nodeBranch.getLeft(), nodeBranch.getRight());
-      return new PendingNodeBranch(commonPath.getPath(), depth + commonPathLength,
-              isRight ? oldBranch : newBranch, isRight ? newBranch : oldBranch);
+    if (!commonPath.getPath().equals(branch.getPath())) {
+      LeafBranch newBranch = new PendingLeafBranch(keyPath, key, data, value);
+      return new PendingNodeBranch(commonPath.getPath(), commonPath.getLength(),
+              isRight ? branch : newBranch, isRight ? newBranch : branch);
     }
 
     if (isRight) {
       return new PendingNodeBranch(nodeBranch.getPath(), nodeBranch.getDepth(),
               nodeBranch.getLeft(),
-              SparseMerkleSumTree.buildTree(nodeBranch.getRight(),
-                      remainingPath.shiftRight(commonPathLength), depth + commonPathLength, key,
-                      data, value));
+              SparseMerkleSumTree.buildTree(nodeBranch.getRight(), keyPath, key, data, value));
     }
 
     return new PendingNodeBranch(nodeBranch.getPath(), nodeBranch.getDepth(),
-            SparseMerkleSumTree.buildTree(nodeBranch.getLeft(),
-                    remainingPath.shiftRight(commonPathLength), depth + commonPathLength, key,
-                    data, value),
+            SparseMerkleSumTree.buildTree(nodeBranch.getLeft(), keyPath, key, data, value),
             nodeBranch.getRight());
   }
 }

src/test/java/org/unicitylabs/sdk/smt/radix/SparseMerkleTreeTest.java

@@ -2,7 +2,11 @@
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.unicitylabs.sdk.api.InclusionCertificate;
+import org.unicitylabs.sdk.api.StateId;
+import org.unicitylabs.sdk.crypto.hash.DataHash;
 import org.unicitylabs.sdk.crypto.hash.HashAlgorithm;
+import org.unicitylabs.sdk.serializer.cbor.CborSerializer;
 
 public class SparseMerkleTreeTest {
 
@@ -16,4 +20,54 @@ void addLeafRejectsNon32ByteKeyOrData() {
     Assertions.assertThrows(NullPointerException.class,
             () -> tree.addLeaf(null, new byte[32]));
   }
+
+  @Test
+  void deepSplitAtDepth255VerifiesWithRegion() throws Exception {
+    SparseMerkleTree tree = new SparseMerkleTree(HashAlgorithm.SHA256);
+
+    byte[] a = new byte[32];
+    byte[] b = new byte[32];
+    b[31] = (byte) 0x80;
+    byte[] valueA = new byte[32];
+    valueA[0] = 1;
+    byte[] valueB = new byte[32];
+    valueB[0] = 2;
+
+    tree.addLeaf(a, valueA);
+    tree.addLeaf(b, valueB);
+    FinalizedNodeBranch root = tree.calculateRoot();
+
+    for (byte[][] entry : new byte[][][]{{a, valueA}, {b, valueB}}) {
+      InclusionCertificate certificate = InclusionCertificate.create(root, entry[0]);
+      StateId key = StateId.fromCbor(CborSerializer.encodeByteString(entry[0]));
+      DataHash value = new DataHash(HashAlgorithm.SHA256, entry[1]);
+      Assertions.assertTrue(certificate.verify(key, value, root.getHash()));
+    }
+  }
+
+  @Test
+  void everyLeafVerifiesThroughNonZeroRegions() throws Exception {
+    int[] firstBytes = {0b10010000, 0b00000000, 0b00010000, 0b10000000, 0b01100000, 0b00010100};
+    byte[][] keys = new byte[firstBytes.length][];
+    byte[][] values = new byte[firstBytes.length][];
+    SparseMerkleTree tree = new SparseMerkleTree(HashAlgorithm.SHA256);
+    for (int i = 0; i < firstBytes.length; i++) {
+      byte[] key = new byte[32];
+      key[0] = (byte) firstBytes[i];
+      byte[] value = new byte[32];
+      value[0] = (byte) (i + 1);
+      keys[i] = key;
+      values[i] = value;
+      tree.addLeaf(key, value);
+    }
+
+    FinalizedNodeBranch root = tree.calculateRoot();
+
+    for (int i = 0; i < keys.length; i++) {
+      InclusionCertificate certificate = InclusionCertificate.create(root, keys[i]);
+      StateId key = StateId.fromCbor(CborSerializer.encodeByteString(keys[i]));
+      DataHash value = new DataHash(HashAlgorithm.SHA256, values[i]);
+      Assertions.assertTrue(certificate.verify(key, value, root.getHash()));
+    }
+  }
 }