diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 7eb3026..23eeb06 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -310,8 +310,10 @@ func TestGetInclusionProofShardMismatch(t *testing.T) { tree := smt.NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, shardingCfg.Child.ShardID) service := newAggregatorServiceForTest(t, shardingCfg, tree) - // Raw 32-byte v2 stateId whose shard-prefix bits don't match shard 4 (=0b100). - invalidShardID := api.RequireNewImprintV2("01" + strings.Repeat("00", api.StateTreeKeyLengthBytes-1)) + // Raw 32-byte v2 stateId whose shard-prefix bits don't match shard 4 + // (=0b100, big-endian bits 1:0 = 00). Byte 0 = 0x80 sets big-endian bit 0, + // so it routes to a different shard. + invalidShardID := api.RequireNewImprintV2("80" + strings.Repeat("00", api.StateTreeKeyLengthBytes-1)) _, err := service.GetInclusionProofV2(context.Background(), &api.GetInclusionProofRequestV2{StateID: invalidShardID}) require.Error(t, err) assert.Contains(t, err.Error(), "state ID validation failed") diff --git a/internal/signing/certification_request_validator_test.go b/internal/signing/certification_request_validator_test.go index f428ff6..a103ac9 100644 --- a/internal/signing/certification_request_validator_test.go +++ b/internal/signing/certification_request_validator_test.go @@ -161,67 +161,63 @@ func TestValidator_ShardID(t *testing.T) { shardBitmask int match bool }{ + // Big-endian bit ordering (issue #169): shard-prefix bits are read from + // the MSB of byte 0 downward, so a shard selects a contiguous key range + // by the high bits of the first byte. + // === TWO SHARD CONFIG === - // shard1=bitmask 0b10 - // shard2=bitmask 0b11 + // shard1=bitmask 0b10 (big-endian bit 0 = 0) + // shard2=bitmask 0b11 (big-endian bit 0 = 1) - // certification request with key bit 0 = 0 belongs to shard1 + // big-endian bit 0 = 0 (first byte < 0x80) belongs to shard1 {makeShardTestID(0x00, 0x00), 0b10, true}, {makeShardTestID(0x00, 0x00), 0b11, false}, + {makeShardTestID(0x7F, 0x00), 0b10, true}, + {makeShardTestID(0x7F, 0x00), 0b11, false}, - // certification request with key bit 0 = 1 belongs to shard2 - {makeShardTestID(0x01, 0x00), 0b10, false}, - {makeShardTestID(0x01, 0x00), 0b11, true}, - - // certification request with first byte 0b00000010 still belongs to shard1 - {makeShardTestID(0x02, 0x00), 0b10, true}, - {makeShardTestID(0x02, 0x00), 0b11, false}, - - // certification request with first byte 0b00000011 belongs to shard2 - {makeShardTestID(0x03, 0x00), 0b10, false}, - {makeShardTestID(0x03, 0x00), 0b11, true}, - - // certification request with first byte 0b11111111 belongs to shard2 + // big-endian bit 0 = 1 (first byte >= 0x80) belongs to shard2 + {makeShardTestID(0x80, 0x00), 0b10, false}, + {makeShardTestID(0x80, 0x00), 0b11, true}, {makeShardTestID(0xFF, 0x00), 0b10, false}, {makeShardTestID(0xFF, 0x00), 0b11, true}, - // the last byte no longer affects shard routing under LSB-first byte order + // the last byte does not affect shard routing {makeShardTestID(0x00, 0xFF), 0b10, true}, {makeShardTestID(0x00, 0xFF), 0b11, false}, // === END TWO SHARD CONFIG === // === FOUR SHARD CONFIG === - // shard1=0b100 - // shard2=0b110 - // shard3=0b101 - // shard4=0b111 + // shard1=0b100 (big-endian bits 1:0 = 00) + // shard2=0b110 (big-endian bits 1:0 = 01) + // shard3=0b101 (big-endian bits 1:0 = 10) + // shard4=0b111 (big-endian bits 1:0 = 11) - // key bits 1:0 = 00 belong to shard1 + // big-endian bits 1:0 = 00 belong to shard1 {makeShardTestID(0x00, 0x00), 0b111, false}, {makeShardTestID(0x00, 0x00), 0b101, false}, {makeShardTestID(0x00, 0x00), 0b110, false}, {makeShardTestID(0x00, 0x00), 0b100, true}, - // key bits 1:0 = 10 belong to shard2 - {makeShardTestID(0x02, 0x00), 0b111, false}, - {makeShardTestID(0x02, 0x00), 0b100, false}, - {makeShardTestID(0x02, 0x00), 0b101, false}, - {makeShardTestID(0x02, 0x00), 0b110, true}, - - // key bits 1:0 = 01 belong to shard3 - {makeShardTestID(0x01, 0x00), 0b111, false}, - {makeShardTestID(0x01, 0x00), 0b101, true}, - {makeShardTestID(0x01, 0x00), 0b110, false}, - {makeShardTestID(0x01, 0x00), 0b100, false}, - - // key bits 1:0 = 11 belong to shard4 - {makeShardTestID(0x03, 0x00), 0b111, true}, - {makeShardTestID(0x03, 0x00), 0b101, false}, - {makeShardTestID(0x03, 0x00), 0b110, false}, - {makeShardTestID(0x03, 0x00), 0b100, false}, - - // key bits 1:0 = 11 still belong to shard4 when the whole first byte is set + // big-endian bits 1:0 = 01 belong to shard2 + {makeShardTestID(0x40, 0x00), 0b111, false}, + {makeShardTestID(0x40, 0x00), 0b100, false}, + {makeShardTestID(0x40, 0x00), 0b101, false}, + {makeShardTestID(0x40, 0x00), 0b110, true}, + + // big-endian bits 1:0 = 10 belong to shard3 + {makeShardTestID(0x80, 0x00), 0b111, false}, + {makeShardTestID(0x80, 0x00), 0b101, true}, + {makeShardTestID(0x80, 0x00), 0b110, false}, + {makeShardTestID(0x80, 0x00), 0b100, false}, + + // big-endian bits 1:0 = 11 belong to shard4 + {makeShardTestID(0xC0, 0x00), 0b111, true}, + {makeShardTestID(0xC0, 0x00), 0b101, false}, + {makeShardTestID(0xC0, 0x00), 0b110, false}, + {makeShardTestID(0xC0, 0x00), 0b100, false}, + + // bits 1:0 = 11 still belong to shard4 when the whole first byte is set {makeShardTestID(0xFF, 0x00), 0b111, true}, {makeShardTestID(0xFF, 0x00), 0b101, false}, {makeShardTestID(0xFF, 0x00), 0b110, false}, diff --git a/internal/smt/be_reference_anchor_test.go b/internal/smt/be_reference_anchor_test.go new file mode 100644 index 0000000..cd7e804 --- /dev/null +++ b/internal/smt/be_reference_anchor_test.go @@ -0,0 +1,155 @@ +package smt + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/unicitynetwork/aggregator-go/pkg/api" +) + +// This file is an independent, self-contained big-endian SMT reference used to +// anchor the production SMT to the yellowpaper spec (issue #169). It shares no +// code with the production packing/traversal path: bits are read big-endian by +// hand and every hash is computed with crypto/sha256 directly. Agreement +// between this reference and production for a set of trees — including +// convention-asymmetric canary keys — proves the production tree is big-endian +// correct, not merely internally self-consistent. + +type beRefLeaf struct { + key []byte + val []byte +} + +func beRefKeyBit(key []byte, d int) int { return int((key[d/8] >> (7 - uint(d)%8)) & 1) } + +func beRefLeafHash(key, val []byte) []byte { + h := sha256.New() + h.Write([]byte{0x00}) + h.Write(key) + h.Write(val) + return h.Sum(nil) +} + +func beRefNodeHash(depth int, region, left, right []byte) []byte { + h := sha256.New() + h.Write([]byte{0x01, byte(depth)}) + h.Write(region) + h.Write(left) + h.Write(right) + return h.Sum(nil) +} + +func beRefRegion(key []byte, depth int) []byte { + r := make([]byte, 32) + for d := 0; d < depth; d++ { + if beRefKeyBit(key, d) == 1 { + r[d/8] |= 0x80 >> (uint(d) % 8) + } + } + return r +} + +// beRefBuild builds the v6a big-endian radix tree for leaves that all share +// bits [0, startBit) and returns its node hash. Single leaf returns the leaf +// hash (unary passthrough), matching the production root of a ≥1-leaf tree. +func beRefBuild(leaves []beRefLeaf, startBit int) []byte { + if len(leaves) == 1 { + return beRefLeafHash(leaves[0].key, leaves[0].val) + } + split := startBit + for { + b := beRefKeyBit(leaves[0].key, split) + same := true + for _, lf := range leaves { + if beRefKeyBit(lf.key, split) != b { + same = false + break + } + } + if !same { + break + } + split++ + } + var left, right []beRefLeaf + for _, lf := range leaves { + if beRefKeyBit(lf.key, split) == 0 { + left = append(left, lf) + } else { + right = append(right, lf) + } + } + region := beRefRegion(leaves[0].key, split) + return beRefNodeHash(split, region, beRefBuild(left, split+1), beRefBuild(right, split+1)) +} + +func beRefKey(b ...byte) []byte { + key := make([]byte, 32) + copy(key, b) + return key +} + +func TestBigEndianReferenceMatchesProduction(t *testing.T) { + cases := []struct { + name string + leaves []beRefLeaf + }{ + {"shallow_split_bit7", []beRefLeaf{ + {beRefKey(0x00), []byte("left")}, {beRefKey(0x01), []byte("right")}, + }}, + {"deep_split_bit255", []beRefLeaf{ + {beRefKey(), beRefKey(1)}, {beRefKeyLast(0x01), beRefKey(2)}, + }}, + // Asymmetric canary: 0x80 (bit 0) vs 0x01 (bit 7) — a half-flipped + // implementation splits these at a different depth than big-endian. + {"asymmetric_bit0_vs_bit7", []beRefLeaf{ + {beRefKey(0x80), []byte("a")}, {beRefKey(0x01), []byte("b")}, {beRefKey(0x00), []byte("c")}, + }}, + {"multi_leaf", []beRefLeaf{ + {beRefKey(0x00), []byte("value0")}, + {beRefKey(0x04), []byte("value1")}, + {beRefKey(0x01), []byte("value2")}, + {beRefKey(0x00, 0x01), []byte("value3")}, + {beRefKey(0x00, 0x00, 0x01), []byte("value4")}, + }}, + {"js_parity_single_byte_keys", func() []beRefLeaf { + firstBytes := []byte{0b10010000, 0b00000000, 0b00010000, 0b10000000, 0b01100000, 0b00010100} + var ls []beRefLeaf + for i, b := range firstBytes { + ls = append(ls, beRefLeaf{beRefKey(b), []byte(fmt.Sprintf("value%d", i))}) + } + return ls + }()}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tree := NewSparseMerkleTree(api.SHA256, 256) + for _, lf := range c.leaves { + p, err := api.FixedBytesToPath(lf.key, 256) + require.NoError(t, err) + require.NoError(t, tree.AddLeaf(p, lf.val)) + } + want := hex.EncodeToString(beRefBuild(c.leaves, 0)) + require.Equal(t, want, tree.GetRootHashHex(), + "production root must equal independent big-endian reference") + + // Every leaf's inclusion cert must verify against the root. + for _, lf := range c.leaves { + cert, err := tree.GetInclusionCert(lf.key) + require.NoError(t, err) + require.NoError(t, cert.Verify(lf.key, lf.val, tree.GetRootHashRaw(), api.SHA256)) + } + }) + } +} + +func beRefKeyLast(b byte) []byte { + key := make([]byte, 32) + key[31] = b + return key +} diff --git a/internal/smt/disk/hash.go b/internal/smt/disk/hash.go index b3e0c55..130a05a 100644 --- a/internal/smt/disk/hash.go +++ b/internal/smt/disk/hash.go @@ -49,7 +49,7 @@ func HashLeaf(key Key, value []byte) Hash { // HashNode implements the yellowpaper v6a (Appendix C.3.2.2) internal-node // hash: H(0x01 || depth_1B || region_32B || left_hash_32B || right_hash_32B). // The region is the node's absolute key prefix at its bifurcation depth, -// packed LSB-in-byte with all bits at positions >= depth cleared. +// packed big-endian with all bits at positions >= depth cleared. func HashNode(left, right Hash, depth uint8, region PrefixBits) Hash { h := sha256.New() _, _ = h.Write([]byte{0x01, depth}) @@ -63,15 +63,14 @@ func HashNode(left, right Hash, depth uint8, region PrefixBits) Hash { // RegionFromKey packs the depth-bit prefix of a descendant key into the // canonical v6a region encoding: key bits 0..depth-1 in place, all bits at -// positions >= depth cleared. +// positions >= depth cleared. Under big-endian bit ordering the region shares +// the key's byte layout, so it is the key bytes with the depth suffix masked. func RegionFromKey(key Key, depth uint8) PrefixBits { var region PrefixBits d := int(depth) byteLen := (d + 7) / 8 copy(region[:byteLen], key[:byteLen]) - if rem := d % 8; rem != 0 { - region[byteLen-1] &= byte(1<> (7 - d%8)) & 1. This is the +// bounds-checked disk-typed accessor over api.KeyBitBE. func KeyBit(key Key, d int) byte { if d < 0 || d >= KeyBits { panic(fmt.Sprintf("disk smt: key bit index out of range: %d", d)) } - return (key[d/8] >> (uint(d) % 8)) & 1 + return api.KeyBitBE(key[:], d) } diff --git a/internal/smt/disk/model_test.go b/internal/smt/disk/model_test.go index 09a9266..79cdcd4 100644 --- a/internal/smt/disk/model_test.go +++ b/internal/smt/disk/model_test.go @@ -23,18 +23,18 @@ func TestNodeKeyEncodingMatchesRugregatorShape(t *testing.T) { left, err := NewNodeKey(1, PrefixBits{}) require.NoError(t, err) var rightPrefix PrefixBits - rightPrefix[0] = 0x01 + rightPrefix[0] = 0x80 // big-endian: depth-1 direction bit is the MSB of byte 0 right, err := NewNodeKey(1, rightPrefix) require.NoError(t, err) require.Equal(t, []byte{0x01, 0x00, 0x00}, left.Bytes()) - require.Equal(t, []byte{0x01, 0x00, 0x01}, right.Bytes()) + require.Equal(t, []byte{0x01, 0x00, 0x80}, right.Bytes()) var deepPrefix PrefixBits deepPrefix[0] = 0x01 - deepPrefix[1] = 0x01 + deepPrefix[1] = 0x80 // big-endian: bit 8 is the MSB of byte 1 deep, err := NewNodeKey(9, deepPrefix) require.NoError(t, err) - require.Equal(t, []byte{0x09, 0x00, 0x01, 0x01}, deep.Bytes()) + require.Equal(t, []byte{0x09, 0x00, 0x01, 0x80}, deep.Bytes()) parsed, err := ParseNodeKey(deep.Bytes()) require.NoError(t, err) @@ -75,7 +75,7 @@ func TestCompressedPathFromKeyRange(t *testing.T) { require.Equal(t, byte(1), path.BitAt(0)) require.Equal(t, byte(0), path.BitAt(1)) require.Equal(t, byte(1), path.BitAt(2)) - require.Equal(t, []byte{0xa5, 0x01}, path.Bytes()) + require.Equal(t, []byte{0xa5, 0x00}, path.Bytes()) roundTrip, err := NewCompressedPathFromRaw(path.Len(), path.Bytes()) require.NoError(t, err) @@ -118,14 +118,18 @@ func TestHashNodeMatchesMemoryAndGoldenRoot(t *testing.T) { l1 := NewLeaf(k1, v1) l2 := NewLeaf(k2, v2) + // Under big-endian bit ordering, keys 0x01 and 0x02 first diverge at depth 6. + const splitDepth = 6 var left, right *Branch - if KeyBit(k1, 0) == 0 { + if KeyBit(k1, splitDepth) == 0 { left, right = l1, l2 } else { left, right = l2, l1 } - root, err := NewInternal(EmptyPath(), 0, PrefixBits{}, left, right) + path, err := NewCompressedPathFromKeyRange(k1, 0, splitDepth) + require.NoError(t, err) + root, err := NewInternal(path, splitDepth, RegionFromKey(k1, splitDepth), left, right) require.NoError(t, err) got, err := root.HashValue() require.NoError(t, err) @@ -135,7 +139,7 @@ func TestHashNodeMatchesMemoryAndGoldenRoot(t *testing.T) { leafInput{key: k1, value: v1}, leafInput{key: k2, value: v2}, ), got) - require.Equal(t, mustHash(t, "fb0b8b6efbb9861202b4f49ca9f2d596f6698d5645f7545b74caf9d8b5161fcc"), got) + require.Equal(t, mustHash(t, "edf6f7d3bcf43f5b4f70d5de0d7e83f5d6210b3cc685ad062d761c4965b3f449"), got) } func TestLeafSerializationRoundTrip(t *testing.T) { @@ -174,7 +178,7 @@ func TestInternalSerializationRoundTrip(t *testing.T) { require.NoError(t, err) rightHash, err := right.HashValue() require.NoError(t, err) - expected := []byte{TagInternal, 13, 13, 0xa5, 0x01} + expected := []byte{TagInternal, 13, 13, 0xa5, 0x00} expected = append(expected, leftHash[:]...) expected = append(expected, rightHash[:]...) require.Equal(t, expected, encoded) diff --git a/internal/smt/disk/node_key.go b/internal/smt/disk/node_key.go index c5e562b..b7c92dc 100644 --- a/internal/smt/disk/node_key.go +++ b/internal/smt/disk/node_key.go @@ -3,6 +3,8 @@ package disk import ( "bytes" "fmt" + + "github.com/unicitynetwork/aggregator-go/pkg/api" ) const rootNodeKeyDepth = uint16(0xffff) @@ -118,14 +120,5 @@ func (k *NodeKey) clearUnusedPrefixBits() { k.prefix = PrefixBits{} return } - depth := int(k.depth) - byteLen := prefixByteLen(depth) - if byteLen < len(k.prefix) { - for i := byteLen; i < len(k.prefix); i++ { - k.prefix[i] = 0 - } - } - if rem := depth % 8; rem != 0 { - k.prefix[byteLen-1] &= byte(1<= int(p.len) { panic(fmt.Sprintf("disk smt: compressed path bit index out of range: %d", pos)) } - return (p.bits[pos/8] >> (uint(pos) % 8)) & 1 + return api.KeyBitBE(p.bits[:], pos) } func (p CompressedPath) Bytes() []byte { @@ -96,18 +101,10 @@ func (p CompressedPath) Equal(other CompressedPath) bool { } func (p CompressedPath) hasCanonicalUnusedBits() bool { - bitLen := int(p.len) - byteLen := prefixByteLen(bitLen) - for i := byteLen; i < len(p.bits); i++ { - if p.bits[i] != 0 { - return false - } - } - if rem := bitLen % 8; rem != 0 { - mask := byte(1<= disk.KeyBits { panic(fmt.Sprintf("disk SMT persist: prefix bit index out of range: %d", pos)) } - prefix[pos/8] |= 1 << (uint(pos) % 8) + api.SetBitBE(prefix[:], pos) } func firstDivergenceInPath(path disk.CompressedPath, key disk.Key, startBit int) (int, error) { diff --git a/internal/smt/disk/persist/tree_test.go b/internal/smt/disk/persist/tree_test.go index f6dfd9d..50b346d 100644 --- a/internal/smt/disk/persist/tree_test.go +++ b/internal/smt/disk/persist/tree_test.go @@ -3,6 +3,8 @@ package persist import ( + "encoding/binary" + "math/rand" "testing" "github.com/stretchr/testify/require" @@ -322,10 +324,12 @@ func TestSnapshotCommitPersistsNonRootLoadedNodeMovement(t *testing.T) { store := openTestStore(t, dir) tree := openPersistTree(t, store) - k0 := keyWithFirstByte(0x00) // root-left leaf - k1 := keyWithFirstByte(0x07) // root-right subtree, bits 1110... - k2 := keyWithFirstByte(0x0f) // root-right subtree, bits 1111... - k3 := keyWithFirstByte(0x01) // root-right subtree, splits non-root path at bit 1 + // Big-endian first bytes chosen so the tree shape matches the original + // LSB-first fixture (bytes are the bit-reversal of 0x00/0x07/0x0f/0x01). + k0 := keyWithFirstByte(0x00) // root-left leaf (big-endian bit 0 = 0) + k1 := keyWithFirstByte(0xE0) // root-right subtree, big-endian bits 111000... + k2 := keyWithFirstByte(0xF0) // root-right subtree, big-endian bits 111100... + k3 := keyWithFirstByte(0x80) // root-right subtree, splits non-root path at bit 1 v0 := []byte("value-zero") v1 := []byte("value-one") v2 := []byte("value-two") @@ -680,6 +684,67 @@ func TestGetInclusionCertsBatchMatchesSingleAfterReopen(t *testing.T) { } } +// Full-width randomized parity for the persisted proof path: splits land far +// past byte 0, exercising keyPathLess ordering, cert bitmap packing, and +// materialization across commits and a reopen for hash-distributed keys. +func TestSnapshotRandomizedFullWidthParityWithMemory(t *testing.T) { + rng := rand.New(rand.NewSource(42)) + dir := t.TempDir() + store := openTestStore(t, dir) + tree := openPersistTree(t, store) + + leaves := make([]memoryLeaf, 0, 400) + seen := make(map[disk.Key]struct{}) + var root disk.Hash + + for batch := 0; batch < 20; batch++ { + inputs := make([]disk.LeafInput, 0, 20) + for i := 0; i < 20; i++ { + var key disk.Key + for { + _, err := rng.Read(key[:]) + require.NoError(t, err) + if _, ok := seen[key]; !ok { + break + } + } + seen[key] = struct{}{} + value := make([]byte, 8) + binary.LittleEndian.PutUint64(value, uint64(batch*20+i)) + inputs = append(inputs, leafInput(key, value)) + leaves = append(leaves, memoryLeaf{key: key, value: value}) + } + + snapshot, err := tree.CreateSnapshot() + require.NoError(t, err) + result, err := snapshot.AddLeaves(inputs) + require.NoError(t, err) + require.Len(t, result.AcceptedIndexes, len(inputs)) + require.Empty(t, result.DuplicateIndexes) + require.Empty(t, result.Rejected) + require.Equal(t, memoryRootAfterLeaves(t, leaves...), result.CandidateRoot) + require.NoError(t, snapshot.Commit(api.NewBigIntFromUint64(uint64(batch+1)))) + root = result.CandidateRoot + } + + require.NoError(t, store.Close()) + store = openTestStore(t, dir) + defer store.Close() + tree = openPersistTree(t, store) + require.Equal(t, root, tree.RootHash()) + + memoryTree := memoryTreeAfterLeaves(t, leaves...) + for _, leaf := range leaves { + diskCert, err := tree.GetInclusionCert(leaf.key[:]) + require.NoError(t, err) + memoryCert, err := memoryTree.GetInclusionCert(leaf.key[:]) + require.NoError(t, err) + require.Equal(t, memoryCert.Bitmap, diskCert.Bitmap) + require.Equal(t, memoryCert.Siblings, diskCert.Siblings) + require.NoError(t, diskCert.Verify(leaf.key[:], leaf.value, root[:], api.SHA256)) + } +} + func TestGetInclusionCertEmptyTreeReturnsError(t *testing.T) { store := openTestStore(t, t.TempDir()) defer store.Close() diff --git a/internal/smt/disk/rocksstore/store.go b/internal/smt/disk/rocksstore/store.go index 244a44a..f2c2655 100644 --- a/internal/smt/disk/rocksstore/store.go +++ b/internal/smt/disk/rocksstore/store.go @@ -64,7 +64,7 @@ import ( const ( SchemaVersion = "1" - TreeLayout = "yellowpaper-rsmt-sha256-v6a" + TreeLayout = "yellowpaper-rsmt-sha256-v6a-be" KeyBits = "256" maxCInt = int(^uint32(0) >> 1) diff --git a/internal/smt/disk/serde.go b/internal/smt/disk/serde.go index f95452f..2d756d1 100644 --- a/internal/smt/disk/serde.go +++ b/internal/smt/disk/serde.go @@ -3,6 +3,8 @@ package disk import ( "encoding/binary" "fmt" + + "github.com/unicitynetwork/aggregator-go/pkg/api" ) const ( @@ -110,7 +112,7 @@ func RegionForNode(key NodeKey, path CompressedPath) (PrefixBits, error) { region := key.Prefix() for i := 0; i < path.Len(); i++ { if path.BitAt(i) != 0 { - region[(start+i)/8] |= 1 << (uint(start+i) % 8) + api.SetBitBE(region[:], start+i) } } return region, nil diff --git a/internal/smt/disk/tree.go b/internal/smt/disk/tree.go index 8528e5c..18299a7 100644 --- a/internal/smt/disk/tree.go +++ b/internal/smt/disk/tree.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "errors" "fmt" + "math/bits" "runtime" "sort" "sync" @@ -487,32 +488,29 @@ func xorKeys(a, b Key) Key { return out } +// firstSetBitFrom returns the position of the first set bit at or after startBit +// under big-endian bit ordering (bit 0 is the MSB of byte 0), or KeyBits if +// none. The first set bit of the XOR of two keys is their first diverging +// depth, so this must scan most-significant bit first within each byte. func firstSetBitFrom(key Key, startBit int) int { byteIdx := startBit / 8 bitOff := startBit % 8 if byteIdx < KeySize { - masked := key[byteIdx] >> bitOff + // Keep only bits at positions >= startBit within the byte: the low + // (8-bitOff) bits under big-endian ordering. + masked := key[byteIdx] & (0xFF >> uint(bitOff)) if masked != 0 { - return startBit + bitsTrailingZeros8(masked) + return byteIdx*8 + bits.LeadingZeros8(masked) } } for i := byteIdx + 1; i < KeySize; i++ { if key[i] != 0 { - return i*8 + bitsTrailingZeros8(key[i]) + return i*8 + bits.LeadingZeros8(key[i]) } } return KeyBits } -func bitsTrailingZeros8(value byte) int { - for i := 0; i < 8; i++ { - if value&(1<= depth are zero. -// RegionFromKeyBytes is the canonical v6a region packing of an LSB-first key -// prefix (see api.RegionFromKeyBytes). +// big-endian position 7-(i%8) of byte i/8; all bits at positions >= depth are +// zero. RegionFromKeyBytes is the canonical v6a region packing of a big-endian +// key prefix (see api.RegionFromKeyBytes). func RegionFromKeyBytes(key []byte, depth int) []byte { return api.RegionFromKeyBytes(key, depth) } @@ -462,7 +462,7 @@ func regionFromPath(path *big.Int, depth uint8) []byte { } for i := 0; i < int(depth); i++ { if path.Bit(i) != 0 { - region[i/8] |= 1 << (uint(i) % 8) + api.SetBitBE(region, i) } } return region @@ -711,7 +711,7 @@ func (smt *SparseMerkleTree) generateInclusionCertWithLeafValue(hasher *api.Data return smt.generateInclusionCertWithLeafValue(hasher, key, child, cert) } - cert.Bitmap[depth/8] |= 1 << (uint(depth) % 8) + api.SetBitBE(cert.Bitmap[:], depth) if len(sibHash) != api.SiblingSize { return nil, fmt.Errorf("smt: sibling hash unexpected length: got %d, want %d", len(sibHash), api.SiblingSize) } @@ -734,10 +734,10 @@ func (smt *SparseMerkleTree) generateInclusionCertWithLeafValue(hasher *api.Data return nil, fmt.Errorf("smt: reached empty subtree in inclusion cert traversal") } -// keyBit returns bit d of the raw key under LSB-first byte layout. -// Matches api.keyBitAt. +// keyBit returns bit d of the raw key under the yellowpaper big-endian bit +// layout. Matches api.KeyBitBE. func keyBit(key []byte, d int) byte { - return (key[d/8] >> (uint(d) % 8)) & 1 + return api.KeyBitBE(key, d) } // GetLeaf retrieves a leaf by path (for compatibility) diff --git a/internal/smt/smt_test.go b/internal/smt/smt_test.go index 604e26c..791f6e6 100644 --- a/internal/smt/smt_test.go +++ b/internal/smt/smt_test.go @@ -35,7 +35,7 @@ func TestSMTGetRoot(t *testing.T) { smt := NewSparseMerkleTree(api.SHA256, 2) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "64a2f31a60210df058e75a10312c486538f8874e4681de085e3e2d9985b5fd50" + expected := "861b3bf0f6e2b77c9925eec9c4aa9c728d96f09115e94176e2f3e18529292759" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -44,7 +44,7 @@ func TestSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b100), []byte{0x61}) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "737f21207992db605e9f894154720f5fc433ca7f5861dba576720633ad948dbd" + expected := "61853feb35bd5147510ba33ce0c010e540978cd98f3af5a8c28ef9f11cc7ef8b" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -55,7 +55,7 @@ func TestSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b1011), []byte{0x63}) smt.AddLeaf(big.NewInt(0b1111), []byte{0x64}) - expected := "2a937ba2bf8c934fa6474c3aea4d20f88a145425bd4874089ca298d2c10d7907" + expected := "b8e036726b506cad5033c88044f2211475887c560fbf757bafe87470b313bf98" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -73,7 +73,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt := NewChildSparseMerkleTree(api.SHA256, 2, 0b11) smt.AddLeaf(big.NewInt(0b111), []byte{0x62}) - expected := "64a2f31a60210df058e75a10312c486538f8874e4681de085e3e2d9985b5fd50" + expected := "861b3bf0f6e2b77c9925eec9c4aa9c728d96f09115e94176e2f3e18529292759" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -82,7 +82,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10010), []byte{0x61}) smt.AddLeaf(big.NewInt(0b11010), []byte{0x62}) - expected := "9cd027f96658b917f35ebab75d1051853c1cdb3adca3fe980ecb1db40cae6cbc" + expected := "a2b824d4a7914518a64767ea0719c465db2733cba475166df0049e7d572380ff" require.Equal(t, expected, smt.GetRootHashHex()) }) @@ -91,7 +91,7 @@ func TestChildSMTGetRoot(t *testing.T) { smt.AddLeaf(big.NewInt(0b10101), []byte{0x63}) smt.AddLeaf(big.NewInt(0b11101), []byte{0x64}) - expected := "b2f48e753b5e65d184354f831ed64e182d0f5d66ac9340c5541d5c9e15e453ce" + expected := "8e83631398be097051feeddef29d7d1025d6625d189611d45265eb2e70f91e4b" require.Equal(t, expected, smt.GetRootHashHex()) }) } @@ -227,7 +227,7 @@ func TestSMTBatchOperations(t *testing.T) { // TestSMTRootHashRegressionFixture pins an implementation reference root hash // for a fixed leaf set, so refactors cannot accidentally change hash behavior. func TestSMTRootHashRegressionFixture(t *testing.T) { - const expectedRoot = "55470bd5b8f6a8a6ecb1bd669e87d8aedaae19c8b5aaed54aaced61704db7012" + const expectedRoot = "548dd4dbde403a16489d7f394597114f75e881ff1f2273668235dc6ba9618577" leaves := []*Leaf{ NewLeaf(big.NewInt(0b110010000), []byte("value00010000")), // 400 diff --git a/internal/smt/v6a_interop_vectors_test.go b/internal/smt/v6a_interop_vectors_test.go index 26d8659..4222420 100644 --- a/internal/smt/v6a_interop_vectors_test.go +++ b/internal/smt/v6a_interop_vectors_test.go @@ -18,12 +18,12 @@ import ( // // leaf: H(0x00 || key || value) // node: H(0x01 || depth_1B || region_32B || left_32B || right_32B) -// region: key bits 0..depth-1 packed LSB-in-byte into 32 bytes, rest zero +// region: key bits 0..depth-1 packed big-endian into 32 bytes, rest zero // empty tree root: all-zero 32 bytes // -// TestV6AInterop_JSSDKParity asserts the JS SDK's published v6a root for the -// same tree — a cross-implementation anchor in the reverse direction -// (JS-generated, Go-reproduced). +// The roots below are the big-endian (issue #169) construction. They are +// additionally cross-checked structurally against an independent big-endian +// reference implementation in TestBigEndianReferenceMatchesProduction. func interopKey(b ...byte) []byte { key := make([]byte, 32) @@ -51,12 +51,16 @@ func interopLeafHash(key, value []byte) string { return hex.EncodeToString(h.Sum(nil)) } -// TestV6AInterop_JSSDKParity builds the identical sparse tree used by the JS -// SDK v6a test suite (32-byte keys with only the first byte set) and asserts -// the JS SDK's expected root, proving cross-implementation agreement. +// TestV6AInterop_JSSDKParity builds the same sparse tree used by the JS SDK +// big-endian test suite (32-byte keys with only the first byte set) and pins +// the shared cross-implementation root. This value is cross-verified against +// the JS SDK: its "should verify the tree" test (state-transition-sdk-js#135, +// issue #134) computes the identical root imprint +// 0000c854db11...166ede for byte-identical inputs, so Go and JS agree in both +// directions on the big-endian construction. func TestV6AInterop_JSSDKParity(t *testing.T) { - // Keys and values mirror the JS SDK's SparseMerkleTree v6a test - // ("should verify the tree"). + // Keys and values mirror the JS SDK's SparseMerkleTree test + // ("should verify the tree"): leavesSparse with values "value0".."value5". firstBytes := []byte{0b10010000, 0b00000000, 0b00010000, 0b10000000, 0b01100000, 0b00010100} tree := NewSparseMerkleTree(api.SHA256, 256) @@ -64,11 +68,11 @@ func TestV6AInterop_JSSDKParity(t *testing.T) { interopAddLeaf(t, tree, interopKey(b), []byte(fmt.Sprintf("value%d", i)), 256) } - // Expected root published by the JS SDK v6a test (imprint prefix stripped). + // Shared cross-SDK big-endian root (JS SDK sdk-js#135 pins the same value). require.Equal(t, - "cd23fc1265484a7173323cd862b85a61796b8e0af31149944a828e6c1734b846", + "c854db11e92d269e7a4dc558adb201da311604d0bcc9883a8f1f017862166ede", interopRoot(t, tree), - "Go v6a root must match the JS SDK v6a root for the identical tree") + "Go root must match the JS SDK big-endian root for the identical tree") for i, b := range firstBytes { requireCertRoundTrip(t, tree, interopKey(b), []byte(fmt.Sprintf("value%d", i))) @@ -96,7 +100,8 @@ func TestV6AInterop_OneLeaf(t *testing.T) { } func TestV6AInterop_ShallowSplit(t *testing.T) { - // Two keys differing at bit 0: one junction at depth 0 with a zero region. + // Two keys differing at big-endian bit 7 (the LSB of byte 0): one junction + // at depth 7 with a zero region. a := interopKey(0x00) b := interopKey(0x01) @@ -110,11 +115,12 @@ func TestV6AInterop_ShallowSplit(t *testing.T) { } func TestV6AInterop_DeepSplit(t *testing.T) { - // Two keys identical except bit 255 (the high bit of the last byte): - // a single junction at depth 255 whose region is 255 shared zero bits. + // Two keys identical except big-endian bit 255 (the LSB of the last byte): + // a single junction at depth 255 whose region is 255 shared zero bits. Same + // inputs as the JS SDK "deep split at depth 255" test (sdk-js#135). a := interopKey() // all zeros b := interopKey() - b[31] = 0x80 + b[31] = 0x01 valueA := interopKey() valueA[0] = 1 @@ -131,14 +137,14 @@ func TestV6AInterop_DeepSplit(t *testing.T) { } func TestV6AInterop_MultiLeaf(t *testing.T) { - // Five keys producing junctions at several depths, including byte - // boundaries (bits 0, 2, 8, 16). + // Five keys producing junctions at several big-endian depths, including + // cross-byte boundaries. keys := [][]byte{ - interopKey(0x00), // bits: all zero - interopKey(0x04), // diverges at bit 2 - interopKey(0x01), // diverges at bit 0 - interopKey(0x00, 0x01), // diverges at bit 8 - interopKey(0x00, 0x00, 0x01), // diverges at bit 16 + interopKey(0x00), // all zero + interopKey(0x04), // diverges at big-endian bit 5 + interopKey(0x01), // diverges at big-endian bit 7 + interopKey(0x00, 0x01), // diverges at big-endian bit 15 + interopKey(0x00, 0x00, 0x01), // diverges at big-endian bit 23 } tree := NewSparseMerkleTree(api.SHA256, 256) @@ -168,7 +174,7 @@ func requireCertRoundTrip(t *testing.T, tree *SparseMerkleTree, key, value []byt // cross-implementation checks. Per-SDK cross-verification status is tracked on // the issue, not here. const ( - shallowSplitExpectedRoot = "8cc069f48345d8117664a31590eea28dae79cac066a4c457cd467c4d4d2648e2" - deepSplitExpectedRoot = "789f3ba1c3b31402bef371ad3cb8a7a176589648d898cb792303a0e3fe128611" - multiLeafExpectedRoot = "7fe744edd3bfe7e973675d773c49e159fccb54e27aa59972deb703fe466bbf2e" + shallowSplitExpectedRoot = "5a758eb5528a347e00b9c8346a3724afc81a5a061cb4b24217d904b8f31f2744" + deepSplitExpectedRoot = "5ae94a6edf95904ebf8d3acbdd0e77c8a991ce6adb8999105325049613ba5580" + multiLeafExpectedRoot = "aeee01fda31fe042467954e26a5788363db7201bbe49b667cdf06c187a6c586b" ) diff --git a/pkg/api/bits.go b/pkg/api/bits.go new file mode 100644 index 0000000..5e80600 --- /dev/null +++ b/pkg/api/bits.go @@ -0,0 +1,38 @@ +package api + +// Big-endian bit-string helpers (yellowpaper "Radix Sparse Merkle Trees", +// big-endian bit strings). A key/region/bitmap is a 256-bit big-endian bit +// string: bit 0 is the most-significant bit of byte 0, bit 255 the +// least-significant bit of byte 31. Bit d lives in byte d/8 at in-byte +// position 7-(d%8). +// +// Every big-endian bit access in pkg/api and internal/smt routes through these +// three helpers (or the disk-typed wrappers that call them) so a single audit +// point governs the whole bit convention. + +// KeyBitBE returns bit d of a big-endian bit string: +// bit d = (buf[d/8] >> (7 - d%8)) & 1. +func KeyBitBE(buf []byte, d int) byte { + return (buf[d/8] >> (7 - uint(d)%8)) & 1 +} + +// SetBitBE sets bit d of a big-endian bit string in place. +func SetBitBE(buf []byte, d int) { + buf[d/8] |= 0x80 >> (uint(d) % 8) +} + +// ClearSuffixBE zeroes every bit at position >= bitLen in the big-endian bit +// string buf. The boundary byte keeps its high (bitLen%8) bits; all bytes +// wholly beyond bitLen are zeroed. bitLen must be in [0, len(buf)*8]. +func ClearSuffixBE(buf []byte, bitLen int) { + if bitLen < 0 { + bitLen = 0 + } + byteLen := (bitLen + 7) / 8 + if rem := bitLen % 8; rem != 0 && byteLen-1 < len(buf) { + buf[byteLen-1] &= byte(0xFF << (8 - uint(rem))) + } + for i := byteLen; i < len(buf); i++ { + buf[i] = 0 + } +} diff --git a/pkg/api/inclusion_cert.go b/pkg/api/inclusion_cert.go index c65e1cd..b843535 100644 --- a/pkg/api/inclusion_cert.go +++ b/pkg/api/inclusion_cert.go @@ -96,7 +96,7 @@ func (c *InclusionCert) UnmarshalBinary(data []byte) error { // algorithm. // // Parameters: -// - key: 32-byte SMT key, LSB-first layout. +// - key: 32-byte SMT key, big-endian bit layout. // - value: raw leaf value bytes (v2 inclusion proofs use the tx hash). // - expectedRoot: raw 32-byte root hash, taken from UC.IR.h. // - algo: hash algorithm used by the SMT. @@ -137,7 +137,7 @@ func verifyBitmapPath(bitmap *[BitmapSize]byte, siblings [][SiblingSize]byte, ke h := append([]byte(nil), startHash...) j := len(siblings) for d := maxDepth - 1; d >= 0; d-- { - if ((*bitmap)[d/8]>>(uint(d)%8))&1 == 0 { + if KeyBitBE((*bitmap)[:], d) == 0 { continue } if d/8 >= len(key) { @@ -150,7 +150,7 @@ func verifyBitmapPath(bitmap *[BitmapSize]byte, siblings [][SiblingSize]byte, ke sibling := siblings[j][:] hasher.Reset().AddData([]byte{0x01, byte(d)}).AddData(RegionFromKeyBytes(key, d)) - if keyBitAt(key, d) == 1 { + if KeyBitBE(key, d) == 1 { // Descent went right at depth d → sibling is the left child. hasher.AddData(sibling).AddData(h) } else { @@ -242,16 +242,11 @@ func bitmapPopcount(b *[BitmapSize]byte) int { return total } -// keyBitAt returns bit d of key under LSB-first byte layout: -// bit d is bit (d mod 8) of key[d / 8]. Matches PathToFixedBytes / -// FixedBytesToPath in state_id.go. -func keyBitAt(key []byte, d int) byte { - return (key[d/8] >> (uint(d) % 8)) & 1 -} - -// RegionFromKeyBytes packs the depth-bit prefix of an LSB-first SMT key into +// RegionFromKeyBytes packs the depth-bit prefix of a big-endian SMT key into // the canonical v6a 32-byte region encoding: key bits 0..depth-1 in place, -// all bits at positions >= depth cleared. +// all bits at positions >= depth cleared. The region shares the key's byte +// layout, so it is the key bytes with the depth suffix masked off. Keys shorter +// than the depth's byte span leave the uncovered prefix bytes zero. func RegionFromKeyBytes(key []byte, depth int) []byte { region := make([]byte, StateTreeKeyLengthBytes) if depth <= 0 { @@ -260,18 +255,11 @@ func RegionFromKeyBytes(key []byte, depth int) []byte { if depth > StateTreeKeyLengthBits { depth = StateTreeKeyLengthBits } - byteLen := (depth + 7) / 8 - if byteLen > len(key) { - byteLen = len(key) - } - copy(region[:byteLen], key[:byteLen]) - // Mask the byte containing the depth boundary. When the key is shorter - // than the depth's byte span, every copied bit is below depth and no - // masking applies. - if rem := depth % 8; rem != 0 { - if maskByte := (depth - 1) / 8; maskByte < byteLen { - region[maskByte] &= byte(1< len(key) { + n = len(key) } + copy(region[:n], key[:n]) + ClearSuffixBE(region, depth) return region } diff --git a/pkg/api/inclusion_cert_compose.go b/pkg/api/inclusion_cert_compose.go index d2b01c1..d26a578 100644 --- a/pkg/api/inclusion_cert_compose.go +++ b/pkg/api/inclusion_cert_compose.go @@ -86,7 +86,7 @@ func bitmapOverlap(a, b *[BitmapSize]byte) bool { func bitmapDepthRange(bitmap *[BitmapSize]byte) (ok bool, minDepth, maxDepthSeen int) { minDepth = BitmapSize * 8 for depth := 0; depth < BitmapSize*8; depth++ { - if (bitmap[depth/8]>>(uint(depth)%8))&1 == 0 { + if KeyBitBE(bitmap[:], depth) == 0 { continue } if !ok { diff --git a/pkg/api/inclusion_cert_test.go b/pkg/api/inclusion_cert_test.go index 7d26e1c..3fd3759 100644 --- a/pkg/api/inclusion_cert_test.go +++ b/pkg/api/inclusion_cert_test.go @@ -46,7 +46,7 @@ func TestInclusionCertVerify_SingleSiblingAtDepth0(t *testing.T) { root := hashNodeRaw(t, SHA256, 0, key, leafHash, siblingHash) cert := &InclusionCert{} - cert.Bitmap[0] = 0x01 // depth 0 + SetBitBE(cert.Bitmap[:], 0) // depth 0 (big-endian: MSB of byte 0) var s [SiblingSize]byte copy(s[:], siblingHash) cert.Siblings = append(cert.Siblings, s) @@ -62,9 +62,9 @@ func TestInclusionCertVerify_TwoSiblingsRootToLeafWireOrder(t *testing.T) { // siblings[1] at depth 7. Verification consumes from the end: // depth 7 first (sib7), then depth 3 (sib3). - // Key byte 0 = 0b0000_1000 → bit 3 = 1, bit 7 = 0. + // Key byte 0 = 0b0001_0000 → big-endian bit 3 = 1, bit 7 = 0. key := make([]byte, StateTreeKeyLengthBytes) - key[0] = 0b0000_1000 + key[0] = 0b0001_0000 value := []byte("v") sib3 := bytes.Repeat([]byte{0x33}, SiblingSize) @@ -78,7 +78,8 @@ func TestInclusionCertVerify_TwoSiblingsRootToLeafWireOrder(t *testing.T) { root := h3 cert := &InclusionCert{} - cert.Bitmap[0] = 0b1000_1000 // bits 3 and 7 + SetBitBE(cert.Bitmap[:], 3) // big-endian bits 3 and 7 + SetBitBE(cert.Bitmap[:], 7) var s3, s7 [SiblingSize]byte copy(s3[:], sib3) copy(s7[:], sib7) @@ -94,7 +95,7 @@ func TestInclusionCertVerify_WrongSiblingOrderFails(t *testing.T) { // Same setup as the two-sibling test but with siblings swapped // into leaf-to-root order. Must fail. key := make([]byte, StateTreeKeyLengthBytes) - key[0] = 0b0000_1000 + key[0] = 0b0001_0000 value := []byte("v") sib3 := bytes.Repeat([]byte{0x33}, SiblingSize) @@ -106,7 +107,8 @@ func TestInclusionCertVerify_WrongSiblingOrderFails(t *testing.T) { root := h3 cert := &InclusionCert{} - cert.Bitmap[0] = 0b1000_1000 + SetBitBE(cert.Bitmap[:], 3) + SetBitBE(cert.Bitmap[:], 7) var s3, s7 [SiblingSize]byte copy(s3[:], sib3) copy(s7[:], sib7) @@ -125,7 +127,7 @@ func TestInclusionCertVerify_DepthSpanning8Bytes(t *testing.T) { // Set bit 200 of the key so we went right at depth 200. key := make([]byte, StateTreeKeyLengthBytes) - key[depth/8] |= 1 << (depth % 8) + SetBitBE(key, depth) value := []byte("deep") siblingHash := bytes.Repeat([]byte{0x5A}, SiblingSize) @@ -134,7 +136,7 @@ func TestInclusionCertVerify_DepthSpanning8Bytes(t *testing.T) { root := hashNodeRaw(t, SHA256, byte(depth), key, siblingHash, leafHash) cert := &InclusionCert{} - cert.Bitmap[depth/8] = 1 << (depth % 8) + SetBitBE(cert.Bitmap[:], depth) var s [SiblingSize]byte copy(s[:], siblingHash) cert.Siblings = append(cert.Siblings, s) @@ -302,27 +304,6 @@ func TestBitmapPopcount(t *testing.T) { } } -func TestKeyBitAt(t *testing.T) { - key := make([]byte, StateTreeKeyLengthBytes) - key[0] = 0b1010_0101 - key[1] = 0x01 // bit 8 set - key[31] = 0x80 // bit 255 set - - checks := []struct { - pos int - want byte - }{ - {0, 1}, {1, 0}, {2, 1}, {5, 1}, {7, 1}, - {8, 1}, {9, 0}, - {255, 1}, - } - for _, c := range checks { - if got := keyBitAt(key, c.pos); got != c.want { - t.Errorf("keyBitAt(%d) = %d, want %d", c.pos, got, c.want) - } - } -} - func TestExclusionCertRoundTrip(t *testing.T) { cert := &ExclusionCert{} for i := range cert.KL { @@ -401,13 +382,13 @@ func TestComposeInclusionCert_Success(t *testing.T) { parentRoot := hashNodeRaw(t, SHA256, 1, key, childRoot, parentSibling) child := &InclusionCert{} - child.Bitmap[5/8] |= 1 << (5 % 8) + SetBitBE(child.Bitmap[:], 5) var childS [SiblingSize]byte copy(childS[:], childSibling) child.Siblings = append(child.Siblings, childS) parent := &InclusionCert{} - parent.Bitmap[1/8] |= 1 << (1 % 8) + SetBitBE(parent.Bitmap[:], 1) var parentS [SiblingSize]byte copy(parentS[:], parentSibling) parent.Siblings = append(parent.Siblings, parentS) @@ -472,12 +453,12 @@ func TestComposeInclusionCert_RejectsChildRootMismatch(t *testing.T) { func TestComposeInclusionCert_RejectsDepthOverlap(t *testing.T) { child := &InclusionCert{} - child.Bitmap[5/8] |= 1 << (5 % 8) + SetBitBE(child.Bitmap[:], 5) var childS [SiblingSize]byte child.Siblings = append(child.Siblings, childS) parent := &InclusionCert{} - parent.Bitmap[5/8] |= 1 << (5 % 8) + SetBitBE(parent.Bitmap[:], 5) var parentS [SiblingSize]byte parent.Siblings = append(parent.Siblings, parentS) parentBytes, err := parent.MarshalBinary() @@ -498,12 +479,12 @@ func TestComposeInclusionCert_RejectsDepthOverlap(t *testing.T) { func TestComposeInclusionCert_RejectsParentDeeperThanChild(t *testing.T) { child := &InclusionCert{} - child.Bitmap[3/8] |= 1 << (3 % 8) + SetBitBE(child.Bitmap[:], 3) var childS [SiblingSize]byte child.Siblings = append(child.Siblings, childS) parent := &InclusionCert{} - parent.Bitmap[7/8] |= 1 << (7 % 8) + SetBitBE(parent.Bitmap[:], 7) var parentS [SiblingSize]byte parent.Siblings = append(parent.Siblings, parentS) parentBytes, err := parent.MarshalBinary() @@ -535,12 +516,12 @@ func TestRegionFromKeyBytes_ShortKey(t *testing.T) { } } - // Uncapped behavior is unchanged: depth 12 over a full-width key masks - // byte 1 to its low 4 bits. + // Uncapped behavior: under big-endian ordering, depth 12 over a full-width + // key keeps byte 1's high 4 bits. full := make([]byte, 32) full[0], full[1] = 0xFF, 0xFF region = RegionFromKeyBytes(full, 12) - if region[0] != 0xFF || region[1] != 0x0F { - t.Fatalf("full-key region = %#x %#x, want 0xff 0x0f", region[0], region[1]) + if region[0] != 0xFF || region[1] != 0xF0 { + t.Fatalf("full-key region = %#x %#x, want 0xff 0xf0", region[0], region[1]) } } diff --git a/pkg/api/shard_match.go b/pkg/api/shard_match.go index 18fc895..0e6a7c5 100644 --- a/pkg/api/shard_match.go +++ b/pkg/api/shard_match.go @@ -6,9 +6,13 @@ import ( "math/bits" ) -// MatchesShardPrefix checks whether the LSB-first bits of keyBytes match the -// shard prefix defined by shardBitmask. The bitmask encodes a sentinel-prefixed -// shard ID (e.g. 0b100 = shard 0 in a 2-bit tree). keyBytes must be at least +// MatchesShardPrefix checks whether the big-endian key bits of keyBytes match +// the shard prefix defined by shardBitmask. The bitmask encodes a +// sentinel-prefixed shard ID (e.g. 0b100 = shard 0 in a 2-bit tree): bit d of +// the mask is the routing decision at depth d, a path/shard-ID convention that +// is independent of the key's byte layout. The key side is read big-endian +// (bit d = MSB-first bit d), so under the yellowpaper big-endian layout a shard +// selects a contiguous lexicographic key range. keyBytes must be at least // ceil(shardDepth/8) bytes long. func MatchesShardPrefix(keyBytes []byte, shardBitmask int) (bool, error) { shardDepth := bits.Len(uint(shardBitmask)) - 1 @@ -21,7 +25,7 @@ func MatchesShardPrefix(keyBytes []byte, shardBitmask int) (bool, error) { for d := 0; d < shardDepth; d++ { expected := byte((uint(shardBitmask) >> uint(d)) & 1) - actual := (keyBytes[d/8] >> (uint(d) % 8)) & 1 + actual := KeyBitBE(keyBytes, d) if actual != expected { return false, nil } diff --git a/pkg/api/smt.go b/pkg/api/smt.go index 221623a..1fae7c9 100644 --- a/pkg/api/smt.go +++ b/pkg/api/smt.go @@ -185,9 +185,10 @@ func (m *MerkleTreePath) Verify(stateID *big.Int) (*PathVerificationResult, erro } // RegionFromPathBits packs bits 0..depth-1 of a sentinel-prefixed path into -// the canonical v6a 32-byte region encoding (bit i at bit i mod 8 of byte -// i / 8, all bits at positions >= depth zero). The low bits of a path are -// absolutely aligned: bit i is the routing decision at tree depth i. +// the canonical v6a 32-byte region encoding (big-endian: path bit i at in-byte +// position 7-(i%8) of byte i/8, all bits at positions >= depth zero). The low +// bits of a path are absolutely aligned: bit i is the routing decision at tree +// depth i, which under the big-endian bijection equals big-endian key bit i. func RegionFromPathBits(path *big.Int, depth int) []byte { region := make([]byte, StateTreeKeyLengthBytes) if path == nil || depth <= 0 { @@ -198,7 +199,7 @@ func RegionFromPathBits(path *big.Int, depth int) []byte { } for i := 0; i < depth; i++ { if path.Bit(i) != 0 { - region[i/8] |= 1 << (uint(i) % 8) + SetBitBE(region, i) } } return region diff --git a/pkg/api/state_id.go b/pkg/api/state_id.go index 3bcd119..5b8a466 100644 --- a/pkg/api/state_id.go +++ b/pkg/api/state_id.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math/big" + "math/bits" "github.com/unicitynetwork/bft-go-base/types" ) @@ -61,8 +62,9 @@ func (r ImprintV2) GetTreeKey() ([]byte, error) { } // PathToFixedBytes converts a sentinel-prefixed SMT path into fixed-width key bytes. -// Byte order follows the v2 SMT bit layout: -// key bit d is bit (d%8) of key[d/8] (LSB-first across bytes). +// Byte order follows the v2 SMT bit layout (yellowpaper big-endian bit strings): +// key bit d is bit (7 - d%8) of key[d/8] (MSB-first within each byte). This is +// the exact inverse of FixedBytesToPath: path bit d = big-endian key bit d. func PathToFixedBytes(path *big.Int, keyLengthBits int) ([]byte, error) { if keyLengthBits <= 0 { return nil, fmt.Errorf("invalid key length: %d", keyLengthBits) @@ -87,10 +89,11 @@ func PathToFixedBytes(path *big.Int, keyLengthBits int) ([]byte, error) { bePadded := make([]byte, keyLengthBytes) copy(bePadded[keyLengthBytes-len(beKey):], beKey) + // Undo the full 256-bit reversal applied by FixedBytesToPath: reverse the + // byte order and the bits within each byte. out := make([]byte, keyLengthBytes) for i := range out { - // Convert from big-endian integer bytes to LSB-first SMT key byte order. - out[i] = bePadded[keyLengthBytes-1-i] + out[i] = bits.Reverse8(bePadded[keyLengthBytes-1-i]) } return out, nil } @@ -105,22 +108,24 @@ func FixedBytesToPath(key []byte, keyLengthBits int) (*big.Int, error) { return nil, fmt.Errorf("invalid key length in bytes: expected %d, got %d", keyLengthBytes, len(key)) } - // For non-byte-aligned keys, ensure unused high bits are zero in the last - // (highest-index) byte under LSB-first key-byte ordering. + // For non-byte-aligned keys, ensure unused bits are zero. Under big-endian + // bit ordering the used bits are the high bits of the last byte, so the + // unused bits are the low (8-rem) bits. if rem := keyLengthBits % 8; rem != 0 { - mask := byte(0xFF << rem) + mask := byte(0xFF >> rem) if key[keyLengthBytes-1]&mask != 0 { - return nil, fmt.Errorf("invalid key: unused high bits must be zero") + return nil, fmt.Errorf("invalid key: unused low bits must be zero") } } - // Convert from LSB-first SMT key byte order to big-endian integer bytes. - be := make([]byte, keyLengthBytes) - for i := range be { - be[i] = key[keyLengthBytes-1-i] + // Map big-endian key bit d to big.Int bit d via a full 256-bit reversal: + // reverse the byte order and the bits within each byte. + rev := make([]byte, keyLengthBytes) + for i := range rev { + rev[i] = bits.Reverse8(key[keyLengthBytes-1-i]) } - path := new(big.Int).SetBytes(be) + path := new(big.Int).SetBytes(rev) path.SetBit(path, keyLengthBits, 1) return path, nil } diff --git a/pkg/api/state_id_bitorder_test.go b/pkg/api/state_id_bitorder_test.go index 3e8c3f4..25a7d46 100644 --- a/pkg/api/state_id_bitorder_test.go +++ b/pkg/api/state_id_bitorder_test.go @@ -6,19 +6,48 @@ import ( "github.com/stretchr/testify/require" ) -func TestFixedBytesToPath_UsesLSBFirstBitAddressing(t *testing.T) { - key := make([]byte, StateTreeKeyLengthBytes) - key[0] = 0x01 +// TestFixedBytesToPath_UsesBigEndianBitAddressing pins the yellowpaper +// big-endian bijection: key bit d (bit 7-d%8 of byte d/8, MSB-first) maps to +// big.Int path bit d. Canary keys with a single set bit per boundary depth +// distinguish big-endian from LSB-first (which would place these bits at the +// mirrored positions). +func TestFixedBytesToPath_UsesBigEndianBitAddressing(t *testing.T) { + cases := []struct { + name string + byteIdx int + byteVal byte + bit int // expected path bit index + }{ + {"depth0_msb_byte0", 0, 0x80, 0}, + {"depth1_byte0", 0, 0x40, 1}, + {"depth7_lsb_byte0", 0, 0x01, 7}, + {"depth8_msb_byte1", 1, 0x80, 8}, + {"depth200_msb_byte25", 25, 0x80, 200}, + {"depth255_lsb_byte31", 31, 0x01, 255}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + key := make([]byte, StateTreeKeyLengthBytes) + key[c.byteIdx] = c.byteVal - path, err := FixedBytesToPath(key, StateTreeKeyLengthBits) - require.NoError(t, err) + path, err := FixedBytesToPath(key, StateTreeKeyLengthBits) + require.NoError(t, err) - // v2 SMT semantics: depth 0 is bit 0 of byte 0. - require.Equal(t, uint(1), path.Bit(0)) - require.Equal(t, uint(0), path.Bit(248)) + require.Equal(t, uint(1), path.Bit(c.bit), "path bit %d must be set", c.bit) + // No other content bit may be set. + for d := 0; d < StateTreeKeyLengthBits; d++ { + if d == c.bit { + continue + } + require.Equal(t, uint(0), path.Bit(d), "path bit %d must be clear", d) + } + // Sentinel bit at keyLengthBits is always set. + require.Equal(t, uint(1), path.Bit(StateTreeKeyLengthBits)) + }) + } } -func TestPathToFixedBytes_RoundtripLSBFirst(t *testing.T) { +func TestPathToFixedBytes_RoundtripBigEndian(t *testing.T) { key := []byte{ 0x8d, 0x17, 0x23, 0x41, 0x99, 0xfe, 0x00, 0x7c, 0x11, 0xaa, 0x52, 0x02, 0x7f, 0x03, 0x10, 0x20, @@ -34,3 +63,59 @@ func TestPathToFixedBytes_RoundtripLSBFirst(t *testing.T) { require.NoError(t, err) require.Equal(t, key, got) } + +// TestKeyBitBE pins the big-endian bit accessor with an asymmetric canary so a +// half-flipped read cannot pass. +func TestKeyBitBE(t *testing.T) { + key := make([]byte, StateTreeKeyLengthBytes) + key[0] = 0b1010_0101 // MSB-first bits 0..7 = 1,0,1,0,0,1,0,1 + key[1] = 0x80 // bit 8 set (MSB of byte 1) + key[31] = 0x01 // bit 255 set (LSB of byte 31) + + checks := []struct { + pos int + want byte + }{ + {0, 1}, {1, 0}, {2, 1}, {3, 0}, {4, 0}, {5, 1}, {6, 0}, {7, 1}, + {8, 1}, {9, 0}, + {248, 0}, {255, 1}, + } + for _, c := range checks { + if got := KeyBitBE(key, c.pos); got != c.want { + t.Errorf("KeyBitBE(%d) = %d, want %d", c.pos, got, c.want) + } + } +} + +// TestSetBitBE_RoundTripsKeyBitBE ensures the setter and getter agree at every +// boundary position. +func TestSetBitBE_RoundTripsKeyBitBE(t *testing.T) { + for _, d := range []int{0, 1, 7, 8, 9, 200, 254, 255} { + buf := make([]byte, StateTreeKeyLengthBytes) + SetBitBE(buf, d) + require.Equal(t, byte(1), KeyBitBE(buf, d), "bit %d must read back set", d) + if d > 0 { + require.Equal(t, byte(0), KeyBitBE(buf, d-1), "neighbor bit %d must be clear", d-1) + } + } +} + +// TestClearSuffixBE_BoundaryDepths checks the big-endian suffix mask keeps the +// high (bitLen%8) bits of the boundary byte and zeroes everything at or beyond +// bitLen. +func TestClearSuffixBE_BoundaryDepths(t *testing.T) { + for _, depth := range []int{0, 1, 7, 8, 200, 255, 256} { + buf := make([]byte, StateTreeKeyLengthBytes) + for i := range buf { + buf[i] = 0xFF + } + ClearSuffixBE(buf, depth) + for d := 0; d < StateTreeKeyLengthBits; d++ { + want := byte(1) + if d >= depth { + want = 0 + } + require.Equal(t, want, KeyBitBE(buf, d), "depth=%d bit=%d", depth, d) + } + } +}