Switch SMT to big-endian bit ordering

internal/service/service_test.go

@@ -310,8 +310,10 @@ func TestGetInclusionProofShardMismatch(t *testing.T) {
 	tree := smt.NewChildSparseMerkleTree(api.SHA256, api.StateTreeKeyLengthBits, shardingCfg.Child.ShardID)
 	service := newAggregatorServiceForTest(t, shardingCfg, tree)
 
-	// Raw 32-byte v2 stateId whose shard-prefix bits don't match shard 4 (=0b100).
-	invalidShardID := api.RequireNewImprintV2("01" + strings.Repeat("00", api.StateTreeKeyLengthBytes-1))
+	// Raw 32-byte v2 stateId whose shard-prefix bits don't match shard 4
+	// (=0b100, big-endian bits 1:0 = 00). Byte 0 = 0x80 sets big-endian bit 0,
+	// so it routes to a different shard.
+	invalidShardID := api.RequireNewImprintV2("80" + strings.Repeat("00", api.StateTreeKeyLengthBytes-1))
 	_, err := service.GetInclusionProofV2(context.Background(), &api.GetInclusionProofRequestV2{StateID: invalidShardID})
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "state ID validation failed")

internal/signing/certification_request_validator_test.go

@@ -161,67 +161,63 @@ func TestValidator_ShardID(t *testing.T) {
 		shardBitmask int
 		match        bool
 	}{
+		// Big-endian bit ordering (issue #169): shard-prefix bits are read from
+		// the MSB of byte 0 downward, so a shard selects a contiguous key range
+		// by the high bits of the first byte.
+
 		// === TWO SHARD CONFIG ===
-		// shard1=bitmask 0b10
-		// shard2=bitmask 0b11
+		// shard1=bitmask 0b10 (big-endian bit 0 = 0)
+		// shard2=bitmask 0b11 (big-endian bit 0 = 1)
 
-		// certification request with key bit 0 = 0 belongs to shard1
+		// big-endian bit 0 = 0 (first byte < 0x80) belongs to shard1
 		{makeShardTestID(0x00, 0x00), 0b10, true},
 		{makeShardTestID(0x00, 0x00), 0b11, false},
+		{makeShardTestID(0x7F, 0x00), 0b10, true},
+		{makeShardTestID(0x7F, 0x00), 0b11, false},
 
-		// certification request with key bit 0 = 1 belongs to shard2
-		{makeShardTestID(0x01, 0x00), 0b10, false},
-		{makeShardTestID(0x01, 0x00), 0b11, true},
-
-		// certification request with first byte 0b00000010 still belongs to shard1
-		{makeShardTestID(0x02, 0x00), 0b10, true},
-		{makeShardTestID(0x02, 0x00), 0b11, false},
-
-		// certification request with first byte 0b00000011 belongs to shard2
-		{makeShardTestID(0x03, 0x00), 0b10, false},
-		{makeShardTestID(0x03, 0x00), 0b11, true},
-
-		// certification request with first byte 0b11111111 belongs to shard2
+		// big-endian bit 0 = 1 (first byte >= 0x80) belongs to shard2
+		{makeShardTestID(0x80, 0x00), 0b10, false},
+		{makeShardTestID(0x80, 0x00), 0b11, true},
 		{makeShardTestID(0xFF, 0x00), 0b10, false},
 		{makeShardTestID(0xFF, 0x00), 0b11, true},
 
-		// the last byte no longer affects shard routing under LSB-first byte order
+		// the last byte does not affect shard routing
 		{makeShardTestID(0x00, 0xFF), 0b10, true},
 		{makeShardTestID(0x00, 0xFF), 0b11, false},
 
 		// === END TWO SHARD CONFIG ===
 
 		// === FOUR SHARD CONFIG ===
-		// shard1=0b100
-		// shard2=0b110
-		// shard3=0b101
-		// shard4=0b111
+		// shard1=0b100 (big-endian bits 1:0 = 00)
+		// shard2=0b110 (big-endian bits 1:0 = 01)
+		// shard3=0b101 (big-endian bits 1:0 = 10)
+		// shard4=0b111 (big-endian bits 1:0 = 11)
 
-		// key bits 1:0 = 00 belong to shard1
+		// big-endian bits 1:0 = 00 belong to shard1
 		{makeShardTestID(0x00, 0x00), 0b111, false},
 		{makeShardTestID(0x00, 0x00), 0b101, false},
 		{makeShardTestID(0x00, 0x00), 0b110, false},
 		{makeShardTestID(0x00, 0x00), 0b100, true},
 
-		// key bits 1:0 = 10 belong to shard2
-		{makeShardTestID(0x02, 0x00), 0b111, false},
-		{makeShardTestID(0x02, 0x00), 0b100, false},
-		{makeShardTestID(0x02, 0x00), 0b101, false},
-		{makeShardTestID(0x02, 0x00), 0b110, true},
-
-		// key bits 1:0 = 01 belong to shard3
-		{makeShardTestID(0x01, 0x00), 0b111, false},
-		{makeShardTestID(0x01, 0x00), 0b101, true},
-		{makeShardTestID(0x01, 0x00), 0b110, false},
-		{makeShardTestID(0x01, 0x00), 0b100, false},
-
-		// key bits 1:0 = 11 belong to shard4
-		{makeShardTestID(0x03, 0x00), 0b111, true},
-		{makeShardTestID(0x03, 0x00), 0b101, false},
-		{makeShardTestID(0x03, 0x00), 0b110, false},
-		{makeShardTestID(0x03, 0x00), 0b100, false},
-
-		// key bits 1:0 = 11 still belong to shard4 when the whole first byte is set
+		// big-endian bits 1:0 = 01 belong to shard2
+		{makeShardTestID(0x40, 0x00), 0b111, false},
+		{makeShardTestID(0x40, 0x00), 0b100, false},
+		{makeShardTestID(0x40, 0x00), 0b101, false},
+		{makeShardTestID(0x40, 0x00), 0b110, true},
+
+		// big-endian bits 1:0 = 10 belong to shard3
+		{makeShardTestID(0x80, 0x00), 0b111, false},
+		{makeShardTestID(0x80, 0x00), 0b101, true},
+		{makeShardTestID(0x80, 0x00), 0b110, false},
+		{makeShardTestID(0x80, 0x00), 0b100, false},
+
+		// big-endian bits 1:0 = 11 belong to shard4
+		{makeShardTestID(0xC0, 0x00), 0b111, true},
+		{makeShardTestID(0xC0, 0x00), 0b101, false},
+		{makeShardTestID(0xC0, 0x00), 0b110, false},
+		{makeShardTestID(0xC0, 0x00), 0b100, false},
+
+		// bits 1:0 = 11 still belong to shard4 when the whole first byte is set
 		{makeShardTestID(0xFF, 0x00), 0b111, true},
 		{makeShardTestID(0xFF, 0x00), 0b101, false},
 		{makeShardTestID(0xFF, 0x00), 0b110, false},

internal/smt/be_reference_anchor_test.go

@@ -0,0 +1,155 @@
+package smt
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/unicitynetwork/aggregator-go/pkg/api"
+)
+
+// This file is an independent, self-contained big-endian SMT reference used to
+// anchor the production SMT to the yellowpaper spec (issue #169). It shares no
+// code with the production packing/traversal path: bits are read big-endian by
+// hand and every hash is computed with crypto/sha256 directly. Agreement
+// between this reference and production for a set of trees — including
+// convention-asymmetric canary keys — proves the production tree is big-endian
+// correct, not merely internally self-consistent.
+
+type beRefLeaf struct {
+	key []byte
+	val []byte
+}
+
+func beRefKeyBit(key []byte, d int) int { return int((key[d/8] >> (7 - uint(d)%8)) & 1) }
+
+func beRefLeafHash(key, val []byte) []byte {
+	h := sha256.New()
+	h.Write([]byte{0x00})
+	h.Write(key)
+	h.Write(val)
+	return h.Sum(nil)
+}
+
+func beRefNodeHash(depth int, region, left, right []byte) []byte {
+	h := sha256.New()
+	h.Write([]byte{0x01, byte(depth)})
+	h.Write(region)
+	h.Write(left)
+	h.Write(right)
+	return h.Sum(nil)
+}
+
+func beRefRegion(key []byte, depth int) []byte {
+	r := make([]byte, 32)
+	for d := 0; d < depth; d++ {
+		if beRefKeyBit(key, d) == 1 {
+			r[d/8] |= 0x80 >> (uint(d) % 8)
+		}
+	}
+	return r
+}
+
+// beRefBuild builds the v6a big-endian radix tree for leaves that all share
+// bits [0, startBit) and returns its node hash. Single leaf returns the leaf
+// hash (unary passthrough), matching the production root of a ≥1-leaf tree.
+func beRefBuild(leaves []beRefLeaf, startBit int) []byte {
+	if len(leaves) == 1 {
+		return beRefLeafHash(leaves[0].key, leaves[0].val)
+	}
+	split := startBit
+	for {
+		b := beRefKeyBit(leaves[0].key, split)
+		same := true
+		for _, lf := range leaves {
+			if beRefKeyBit(lf.key, split) != b {
+				same = false
+				break
+			}
+		}
+		if !same {
+			break
+		}
+		split++
+	}
+	var left, right []beRefLeaf
+	for _, lf := range leaves {
+		if beRefKeyBit(lf.key, split) == 0 {
+			left = append(left, lf)
+		} else {
+			right = append(right, lf)
+		}
+	}
+	region := beRefRegion(leaves[0].key, split)
+	return beRefNodeHash(split, region, beRefBuild(left, split+1), beRefBuild(right, split+1))
+}
+
+func beRefKey(b ...byte) []byte {
+	key := make([]byte, 32)
+	copy(key, b)
+	return key
+}
+
+func TestBigEndianReferenceMatchesProduction(t *testing.T) {
+	cases := []struct {
+		name   string
+		leaves []beRefLeaf
+	}{
+		{"shallow_split_bit7", []beRefLeaf{
+			{beRefKey(0x00), []byte("left")}, {beRefKey(0x01), []byte("right")},
+		}},
+		{"deep_split_bit255", []beRefLeaf{
+			{beRefKey(), beRefKey(1)}, {beRefKeyLast(0x01), beRefKey(2)},
+		}},
+		// Asymmetric canary: 0x80 (bit 0) vs 0x01 (bit 7) — a half-flipped
+		// implementation splits these at a different depth than big-endian.
+		{"asymmetric_bit0_vs_bit7", []beRefLeaf{
+			{beRefKey(0x80), []byte("a")}, {beRefKey(0x01), []byte("b")}, {beRefKey(0x00), []byte("c")},
+		}},
+		{"multi_leaf", []beRefLeaf{
+			{beRefKey(0x00), []byte("value0")},
+			{beRefKey(0x04), []byte("value1")},
+			{beRefKey(0x01), []byte("value2")},
+			{beRefKey(0x00, 0x01), []byte("value3")},
+			{beRefKey(0x00, 0x00, 0x01), []byte("value4")},
+		}},
+		{"js_parity_single_byte_keys", func() []beRefLeaf {
+			firstBytes := []byte{0b10010000, 0b00000000, 0b00010000, 0b10000000, 0b01100000, 0b00010100}
+			var ls []beRefLeaf
+			for i, b := range firstBytes {
+				ls = append(ls, beRefLeaf{beRefKey(b), []byte(fmt.Sprintf("value%d", i))})
+			}
+			return ls
+		}()},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			tree := NewSparseMerkleTree(api.SHA256, 256)
+			for _, lf := range c.leaves {
+				p, err := api.FixedBytesToPath(lf.key, 256)
+				require.NoError(t, err)
+				require.NoError(t, tree.AddLeaf(p, lf.val))
+			}
+			want := hex.EncodeToString(beRefBuild(c.leaves, 0))
+			require.Equal(t, want, tree.GetRootHashHex(),
+				"production root must equal independent big-endian reference")
+
+			// Every leaf's inclusion cert must verify against the root.
+			for _, lf := range c.leaves {
+				cert, err := tree.GetInclusionCert(lf.key)
+				require.NoError(t, err)
+				require.NoError(t, cert.Verify(lf.key, lf.val, tree.GetRootHashRaw(), api.SHA256))
+			}
+		})
+	}
+}
+
+func beRefKeyLast(b byte) []byte {
+	key := make([]byte, 32)
+	key[31] = b
+	return key
+}

internal/smt/disk/hash.go

@@ -49,7 +49,7 @@ func HashLeaf(key Key, value []byte) Hash {
 // HashNode implements the yellowpaper v6a (Appendix C.3.2.2) internal-node
 // hash: H(0x01 || depth_1B || region_32B || left_hash_32B || right_hash_32B).
 // The region is the node's absolute key prefix at its bifurcation depth,
-// packed LSB-in-byte with all bits at positions >= depth cleared.
+// packed big-endian with all bits at positions >= depth cleared.
 func HashNode(left, right Hash, depth uint8, region PrefixBits) Hash {
 	h := sha256.New()
 	_, _ = h.Write([]byte{0x01, depth})
@@ -63,15 +63,14 @@ func HashNode(left, right Hash, depth uint8, region PrefixBits) Hash {
 
 // RegionFromKey packs the depth-bit prefix of a descendant key into the
 // canonical v6a region encoding: key bits 0..depth-1 in place, all bits at
-// positions >= depth cleared.
+// positions >= depth cleared. Under big-endian bit ordering the region shares
+// the key's byte layout, so it is the key bytes with the depth suffix masked.
 func RegionFromKey(key Key, depth uint8) PrefixBits {
 	var region PrefixBits
 	d := int(depth)
 	byteLen := (d + 7) / 8
 	copy(region[:byteLen], key[:byteLen])
-	if rem := d % 8; rem != 0 {
-		region[byteLen-1] &= byte(1<<uint(rem)) - 1
-	}
+	api.ClearSuffixBE(region[:], d)
 	return region
 }
 
@@ -82,10 +81,12 @@ func EmptyRootHash() Hash {
 	return Hash{}
 }
 
-// KeyBit returns bit d of key using the yellowpaper/Go v2 LSB-first key layout.
+// KeyBit returns bit d of key using the yellowpaper big-endian bit layout:
+// bit 0 is the MSB of byte 0, bit d = (key[d/8] >> (7 - d%8)) & 1. This is the
+// bounds-checked disk-typed accessor over api.KeyBitBE.
 func KeyBit(key Key, d int) byte {
 	if d < 0 || d >= KeyBits {
 		panic(fmt.Sprintf("disk smt: key bit index out of range: %d", d))
 	}
-	return (key[d/8] >> (uint(d) % 8)) & 1
+	return api.KeyBitBE(key[:], d)
 }

internal/smt/disk/model_test.go

@@ -23,18 +23,18 @@ func TestNodeKeyEncodingMatchesRugregatorShape(t *testing.T) {
 	left, err := NewNodeKey(1, PrefixBits{})
 	require.NoError(t, err)
 	var rightPrefix PrefixBits
-	rightPrefix[0] = 0x01
+	rightPrefix[0] = 0x80 // big-endian: depth-1 direction bit is the MSB of byte 0
 	right, err := NewNodeKey(1, rightPrefix)
 	require.NoError(t, err)
 	require.Equal(t, []byte{0x01, 0x00, 0x00}, left.Bytes())
-	require.Equal(t, []byte{0x01, 0x00, 0x01}, right.Bytes())
+	require.Equal(t, []byte{0x01, 0x00, 0x80}, right.Bytes())
 
 	var deepPrefix PrefixBits
 	deepPrefix[0] = 0x01
-	deepPrefix[1] = 0x01
+	deepPrefix[1] = 0x80 // big-endian: bit 8 is the MSB of byte 1
 	deep, err := NewNodeKey(9, deepPrefix)
 	require.NoError(t, err)
-	require.Equal(t, []byte{0x09, 0x00, 0x01, 0x01}, deep.Bytes())
+	require.Equal(t, []byte{0x09, 0x00, 0x01, 0x80}, deep.Bytes())
 
 	parsed, err := ParseNodeKey(deep.Bytes())
 	require.NoError(t, err)
@@ -75,7 +75,7 @@ func TestCompressedPathFromKeyRange(t *testing.T) {
 	require.Equal(t, byte(1), path.BitAt(0))
 	require.Equal(t, byte(0), path.BitAt(1))
 	require.Equal(t, byte(1), path.BitAt(2))
-	require.Equal(t, []byte{0xa5, 0x01}, path.Bytes())
+	require.Equal(t, []byte{0xa5, 0x00}, path.Bytes())
 
 	roundTrip, err := NewCompressedPathFromRaw(path.Len(), path.Bytes())
 	require.NoError(t, err)
@@ -118,14 +118,18 @@ func TestHashNodeMatchesMemoryAndGoldenRoot(t *testing.T) {
 	l1 := NewLeaf(k1, v1)
 	l2 := NewLeaf(k2, v2)
 
+	// Under big-endian bit ordering, keys 0x01 and 0x02 first diverge at depth 6.
+	const splitDepth = 6
 	var left, right *Branch
-	if KeyBit(k1, 0) == 0 {
+	if KeyBit(k1, splitDepth) == 0 {
 		left, right = l1, l2
 	} else {
 		left, right = l2, l1
 	}
 
-	root, err := NewInternal(EmptyPath(), 0, PrefixBits{}, left, right)
+	path, err := NewCompressedPathFromKeyRange(k1, 0, splitDepth)
+	require.NoError(t, err)
+	root, err := NewInternal(path, splitDepth, RegionFromKey(k1, splitDepth), left, right)
 	require.NoError(t, err)
 	got, err := root.HashValue()
 	require.NoError(t, err)
@@ -135,7 +139,7 @@ func TestHashNodeMatchesMemoryAndGoldenRoot(t *testing.T) {
 		leafInput{key: k1, value: v1},
 		leafInput{key: k2, value: v2},
 	), got)
-	require.Equal(t, mustHash(t, "fb0b8b6efbb9861202b4f49ca9f2d596f6698d5645f7545b74caf9d8b5161fcc"), got)
+	require.Equal(t, mustHash(t, "edf6f7d3bcf43f5b4f70d5de0d7e83f5d6210b3cc685ad062d761c4965b3f449"), got)
 }
 
 func TestLeafSerializationRoundTrip(t *testing.T) {
@@ -174,7 +178,7 @@ func TestInternalSerializationRoundTrip(t *testing.T) {
 	require.NoError(t, err)
 	rightHash, err := right.HashValue()
 	require.NoError(t, err)
-	expected := []byte{TagInternal, 13, 13, 0xa5, 0x01}
+	expected := []byte{TagInternal, 13, 13, 0xa5, 0x00}
 	expected = append(expected, leftHash[:]...)
 	expected = append(expected, rightHash[:]...)
 	require.Equal(t, expected, encoded)