diff --git a/dot/types/block.go b/dot/types/block.go index c93b0cb801..93a0a31811 100644 --- a/dot/types/block.go +++ b/dot/types/block.go @@ -17,7 +17,7 @@ type Block struct { } // NewBlockFromGeneric returns a new Block from a generic block -func NewBlockFromGeneric[N runtime.Number, H runtime.Hash, E runtime.Extrinsic](gb runtime.Block[N, H, E]) ( +func NewBlockFromGeneric[N runtime.Number, H runtime.Hash, E runtime.Extrinsic, Header runtime.Header[N, H]](gb runtime.Block[H, N, E, Header]) ( *Block, error) { header, err := NewHeaderFromGeneric(gb.Header()) if err != nil { diff --git a/dot/types/block_test.go b/dot/types/block_test.go index 711eb977c9..2c2f47ee07 100644 --- a/dot/types/block_test.go +++ b/dot/types/block_test.go @@ -154,19 +154,19 @@ func TestNewBlockFromGeneric(t *testing.T) { digest := runtime.Digest{ Logs: []runtime.DigestItem{ - runtime.NewDigestItem(runtime.Consensus{ + runtime.DigestItemConsensus{ ConsensusEngineID: runtime.ConsensusEngineID{'B', 'E', 'E', 'F'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.Seal{ + }, + runtime.DigestItemSeal{ ConsensusEngineID: runtime.ConsensusEngineID{'S', 'E', 'A', 'L'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.PreRuntime{ + }, + runtime.DigestItemPreRuntime{ ConsensusEngineID: runtime.ConsensusEngineID{'B', 'A', 'B', 'E'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.RuntimeEnvironmentUpdated{}), + }, + runtime.DigestItemRuntimeEnvironmentUpdated{}, }, } diff --git a/dot/types/digest.go b/dot/types/digest.go index 659ac32958..34d3b1786f 100644 --- a/dot/types/digest.go +++ b/dot/types/digest.go @@ -103,36 +103,31 @@ type Digest []DigestItem func NewDigestFromGeneric(gd runtime.Digest) (Digest, error) { newDigest := Digest{} for _, log := range gd.Logs { - value, err := log.Value() - if err != nil { - return nil, err - } - var digest any - switch v := value.(type) { - case runtime.PreRuntime: + switch v := log.(type) { + case runtime.DigestItemPreRuntime: digest = PreRuntimeDigest{ ConsensusEngineID: ConsensusEngineID(v.ConsensusEngineID), Data: v.Bytes, } - case runtime.Consensus: + case runtime.DigestItemConsensus: digest = ConsensusDigest{ ConsensusEngineID: ConsensusEngineID(v.ConsensusEngineID), Data: v.Bytes, } - case runtime.Seal: + case runtime.DigestItemSeal: digest = SealDigest{ ConsensusEngineID: ConsensusEngineID(v.ConsensusEngineID), Data: v.Bytes, } - case runtime.RuntimeEnvironmentUpdated: + case runtime.DigestItemRuntimeEnvironmentUpdated: digest = RuntimeEnvironmentUpdated{} default: return nil, fmt.Errorf("unsupported type") } - err = newDigest.Add(digest) + err := newDigest.Add(digest) if err != nil { return nil, err } diff --git a/dot/types/digest_test.go b/dot/types/digest_test.go index 3220751aab..8f94e6d5d0 100644 --- a/dot/types/digest_test.go +++ b/dot/types/digest_test.go @@ -249,19 +249,19 @@ func TestSealDigest(t *testing.T) { func TestFromGenericDigest(t *testing.T) { digest := runtime.Digest{ Logs: []runtime.DigestItem{ - runtime.NewDigestItem(runtime.Consensus{ + runtime.DigestItemConsensus{ ConsensusEngineID: runtime.ConsensusEngineID{'B', 'E', 'E', 'F'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.Seal{ + }, + runtime.DigestItemSeal{ ConsensusEngineID: runtime.ConsensusEngineID{'S', 'E', 'A', 'L'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.PreRuntime{ + }, + runtime.DigestItemPreRuntime{ ConsensusEngineID: runtime.ConsensusEngineID{'B', 'A', 'B', 'E'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.RuntimeEnvironmentUpdated{}), + }, + runtime.DigestItemRuntimeEnvironmentUpdated{}, }, } diff --git a/dot/types/header_test.go b/dot/types/header_test.go index a4f367a140..05101d7259 100644 --- a/dot/types/header_test.go +++ b/dot/types/header_test.go @@ -114,19 +114,19 @@ func TestFromGenericHeader(t *testing.T) { t.Run("successful_conversion", func(t *testing.T) { digest := runtime.Digest{ Logs: []runtime.DigestItem{ - runtime.NewDigestItem(runtime.Consensus{ + runtime.DigestItemConsensus{ ConsensusEngineID: runtime.ConsensusEngineID{'B', 'E', 'E', 'F'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.Seal{ + }, + runtime.DigestItemSeal{ ConsensusEngineID: runtime.ConsensusEngineID{'S', 'E', 'A', 'L'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.PreRuntime{ + }, + runtime.DigestItemPreRuntime{ ConsensusEngineID: runtime.ConsensusEngineID{'B', 'A', 'B', 'E'}, Bytes: []byte("test"), - }), - runtime.NewDigestItem(runtime.RuntimeEnvironmentUpdated{}), + }, + runtime.DigestItemRuntimeEnvironmentUpdated{}, }, } diff --git a/internal/client/adapter/client_adapter.go b/internal/client/adapter/client_adapter.go index d764cf7720..db822b2c08 100644 --- a/internal/client/adapter/client_adapter.go +++ b/internal/client/adapter/client_adapter.go @@ -78,7 +78,7 @@ func (ca *ClientAdapter[H, Hasher, N, E, Header]) BestBlock() (*types.Block, err return nil, nil } - return types.NewBlockFromGeneric(signedBlock.Block) + return types.NewBlockFromGeneric[N, H, E](signedBlock.Block) } func (ca *ClientAdapter[H, Hasher, N, E, Header]) BestBlockHash() common.Hash { @@ -144,7 +144,7 @@ func (ca *ClientAdapter[H, Hasher, N, E, Header]) GetBlockByHash(bhash common.Ha return nil, nil } - return types.NewBlockFromGeneric(block.Block) + return types.NewBlockFromGeneric[N, H, E](block.Block) } func (ca *ClientAdapter[H, Hasher, N, E, Header]) GetBlockByNumber(blockNumber uint) (*types.Block, error) { @@ -166,7 +166,7 @@ func (ca *ClientAdapter[H, Hasher, N, E, Header]) GetBlockByNumber(blockNumber u return nil, nil } - return types.NewBlockFromGeneric(signedBlock.Block) + return types.NewBlockFromGeneric[N, H, E](signedBlock.Block) } // GetFinalisedHeader is unimplemented diff --git a/internal/client/adapter/client_adapter_test.go b/internal/client/adapter/client_adapter_test.go index e10e0e1003..581a4670d7 100644 --- a/internal/client/adapter/client_adapter_test.go +++ b/internal/client/adapter/client_adapter_test.go @@ -203,7 +203,7 @@ func TestBlockOps(t *testing.T) { client, _, adapter := setupTest(t) t.Run("best_block_ok", func(t *testing.T) { - expectedBlock, err := types.NewBlockFromGeneric(signedBlock.Block) + expectedBlock, err := types.NewBlockFromGeneric[Number, Hash, Extrinsic](signedBlock.Block) require.NoError(t, err) client.EXPECT().Info().Return(blockchainInfo) @@ -326,7 +326,7 @@ func TestGetBlockByNumber(t *testing.T) { require.NoError(t, err) require.NotNil(t, block) - expectedBlock, err := types.NewBlockFromGeneric(block) + expectedBlock, err := types.NewBlockFromGeneric[Number, Hash, Extrinsic](block) require.NoError(t, err) require.Equal(t, expectedBlock, returnedBlock) diff --git a/internal/client/adapter/mocks/client.go b/internal/client/adapter/mocks/client.go index f405ebe281..e74a3afeee 100644 --- a/internal/client/adapter/mocks/client.go +++ b/internal/client/adapter/mocks/client.go @@ -31,23 +31,23 @@ func (_m *Client[H, Hasher, N, E, Header]) EXPECT() *Client_Expecter[H, Hasher, } // Block provides a mock function with given fields: hash -func (_m *Client[H, Hasher, N, E, Header]) Block(hash H) (*generic.SignedBlock[N, H, Hasher, E], error) { +func (_m *Client[H, Hasher, N, E, Header]) Block(hash H) (*generic.SignedBlock[N, H, Hasher, E, Header], error) { ret := _m.Called(hash) if len(ret) == 0 { panic("no return value specified for Block") } - var r0 *generic.SignedBlock[N, H, Hasher, E] + var r0 *generic.SignedBlock[N, H, Hasher, E, Header] var r1 error - if rf, ok := ret.Get(0).(func(H) (*generic.SignedBlock[N, H, Hasher, E], error)); ok { + if rf, ok := ret.Get(0).(func(H) (*generic.SignedBlock[N, H, Hasher, E, Header], error)); ok { return rf(hash) } - if rf, ok := ret.Get(0).(func(H) *generic.SignedBlock[N, H, Hasher, E]); ok { + if rf, ok := ret.Get(0).(func(H) *generic.SignedBlock[N, H, Hasher, E, Header]); ok { r0 = rf(hash) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*generic.SignedBlock[N, H, Hasher, E]) + r0 = ret.Get(0).(*generic.SignedBlock[N, H, Hasher, E, Header]) } } @@ -78,12 +78,12 @@ func (_c *Client_Block_Call[H, Hasher, N, E, Header]) Run(run func(hash H)) *Cli return _c } -func (_c *Client_Block_Call[H, Hasher, N, E, Header]) Return(_a0 *generic.SignedBlock[N, H, Hasher, E], _a1 error) *Client_Block_Call[H, Hasher, N, E, Header] { +func (_c *Client_Block_Call[H, Hasher, N, E, Header]) Return(_a0 *generic.SignedBlock[N, H, Hasher, E, Header], _a1 error) *Client_Block_Call[H, Hasher, N, E, Header] { _c.Call.Return(_a0, _a1) return _c } -func (_c *Client_Block_Call[H, Hasher, N, E, Header]) RunAndReturn(run func(H) (*generic.SignedBlock[N, H, Hasher, E], error)) *Client_Block_Call[H, Hasher, N, E, Header] { +func (_c *Client_Block_Call[H, Hasher, N, E, Header]) RunAndReturn(run func(H) (*generic.SignedBlock[N, H, Hasher, E, Header], error)) *Client_Block_Call[H, Hasher, N, E, Header] { _c.Call.Return(run) return _c } diff --git a/internal/client/api/backend.go b/internal/client/api/backend.go index 99e7cf0bfe..530135995e 100644 --- a/internal/client/api/backend.go +++ b/internal/client/api/backend.go @@ -9,6 +9,7 @@ import ( "github.com/ChainSafe/gossamer/internal/primitives/blockchain" "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" "github.com/ChainSafe/gossamer/internal/primitives/core/offchain" + "github.com/ChainSafe/gossamer/internal/primitives/kv" "github.com/ChainSafe/gossamer/internal/primitives/runtime" statemachine "github.com/ChainSafe/gossamer/internal/primitives/state-machine" "github.com/ChainSafe/gossamer/internal/primitives/state-machine/overlayedchanges" @@ -237,10 +238,7 @@ type Finalizer[ } // KeyValue is used in [AuxStore.InsertAux]. Key and Value should not be nil. -type KeyValue struct { - Key []byte - Value []byte -} +type KeyValue = kv.KeyValue // AuxStore provides access to an auxiliary database. // @@ -256,6 +254,109 @@ type AuxStore interface { GetAux(key []byte) ([]byte, error) } +// / An `Iterator` that iterates keys in a given block under a prefix. +// pub struct KeysIter +// where +// +// State: StateBackend>, +// Block: BlockT, +// +// { +// inner: >>::RawIter, +// state: State, +// } +type KeysIter[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]] struct { + inner statemachine.StorageIterator[H, Hasher] + state statemachine.Backend[H, Hasher] +} + +// / An `Iterator` that iterates keys and values in a given block under a prefix. +// pub struct PairsIter +// where +// +// State: StateBackend>, +// Block: BlockT, +// +// { +// inner: >>::RawIter, +// state: State, +// } +type PairsIter[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]] struct { + inner statemachine.StorageIterator[H, Hasher] + state statemachine.Backend[H, Hasher] +} + +// / Provides access to storage primitives +// pub trait StorageProvider> { +type StorageProvider[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]] interface { + /// Given a block's `Hash` and a key, return the value under the key in that block. + // fn storage( + // &self, + // hash: Block::Hash, + // key: &StorageKey, + // ) -> sp_blockchain::Result>; + Storage(hash H, key storage.StorageKey) (*storage.StorageData, error) + + /// Given a block's `Hash` and a key, return the value under the hash in that block. + // fn storage_hash( + // &self, + // hash: Block::Hash, + // key: &StorageKey, + // ) -> sp_blockchain::Result>; + StorageHash(hash H, key storage.StorageKey) (*H, error) + + /// Given a block's `Hash` and a key prefix, returns a `KeysIter` iterates matching storage + /// keys in that block. + // fn storage_keys( + // &self, + // hash: Block::Hash, + // prefix: Option<&StorageKey>, + // start_key: Option<&StorageKey>, + // ) -> sp_blockchain::Result>; + StorageKeys(hash H, prefix *storage.StorageKey, startKey *storage.StorageKey) (KeysIter[H, N, Hasher], error) + + /// Given a block's `Hash` and a key prefix, returns an iterator over the storage keys and + /// values in that block. + // fn storage_pairs( + // &self, + // hash: ::Hash, + // prefix: Option<&StorageKey>, + // start_key: Option<&StorageKey>, + // ) -> sp_blockchain::Result>; + StoragePairs(hash H, prefix *storage.StorageKey, startKey *storage.StorageKey) (PairsIter[H, N, Hasher], error) + + /// Given a block's `Hash`, a key and a child storage key, return the value under the key in + /// that block. + // fn child_storage( + // &self, + // hash: Block::Hash, + // child_info: &ChildInfo, + // key: &StorageKey, + // ) -> sp_blockchain::Result>; + ChildStorage(hash H, childInfo storage.ChildInfo, key storage.StorageKey) (*storage.StorageData, error) + + // /// Given a block's `Hash` and a key `prefix` and a child storage key, + // /// returns a `KeysIter` that iterates matching storage keys in that block. + // fn child_storage_keys( + // &self, + // hash: Block::Hash, + // child_info: ChildInfo, + // prefix: Option<&StorageKey>, + // start_key: Option<&StorageKey>, + // ) -> sp_blockchain::Result>; + ChildStorageKeys(hash H, childInfo storage.ChildInfo, prefix *storage.StorageKey, startKey *storage.StorageKey) (KeysIter[H, N, Hasher], error) + + // /// Given a block's `Hash`, a key and a child storage key, return the hash under the key in that + // /// block. + // fn child_storage_hash( + // &self, + // hash: Block::Hash, + // child_info: &ChildInfo, + // key: &StorageKey, + // ) -> sp_blockchain::Result>; + ChildStorageHash(hash H, childInfo storage.ChildInfo, key storage.StorageKey) (*H, error) +} + // Backend is the client backend. // // Manages the data layer. diff --git a/internal/client/api/call_executor.go b/internal/client/api/call_executor.go new file mode 100644 index 0000000000..22ecfa834b --- /dev/null +++ b/internal/client/api/call_executor.go @@ -0,0 +1,58 @@ +package api + +// / Method call executor. +// pub trait CallExecutor: RuntimeVersionOf { +type CallExecutor interface { + // /// Externalities error type. + // type Error: sp_state_machine::Error; + + // /// The backend used by the node. + // type Backend: crate::backend::Backend; + + // /// Returns the [`ExecutionExtensions`]. + // fn execution_extensions(&self) -> &ExecutionExtensions; + + // /// Execute a call to a contract on top of state in a block of given hash. + // /// + // /// No changes are made. + // fn call( + // &self, + // at_hash: B::Hash, + // method: &str, + // call_data: &[u8], + // context: CallContext, + // ) -> Result, sp_blockchain::Error>; + + // /// Execute a contextual call on top of state in a block of a given hash. + // /// + // /// No changes are made. + // /// Before executing the method, passed header is installed as the current header + // /// of the execution context. + // fn contextual_call( + // &self, + // at_hash: B::Hash, + // method: &str, + // call_data: &[u8], + // changes: &RefCell>>, + // proof_recorder: &Option>, + // call_context: CallContext, + // extensions: &RefCell, + // ) -> sp_blockchain::Result>; + + // /// Extract RuntimeVersion of given block + // /// + // /// No changes are made. + // fn runtime_version(&self, at_hash: B::Hash) -> Result; + + // /// Prove the execution of the given `method`. + // /// + // /// No changes are made. + // fn prove_execution( + // + // &self, + // at_hash: B::Hash, + // method: &str, + // call_data: &[u8], + // + // ) -> Result<(Vec, StorageProof), sp_blockchain::Error>; +} diff --git a/internal/client/api/client.go b/internal/client/api/client.go index 0cea08a1dc..6dbf7325f5 100644 --- a/internal/client/api/client.go +++ b/internal/client/api/client.go @@ -24,6 +24,20 @@ type FinalityNotifications[ Header runtime.Header[N, H], ] chan FinalityNotification[H, N, Header] +// / Expected hashes of blocks at given heights. +// / +// / This may be used as chain spec extension to set trusted checkpoints, i.e. +// / the client will refuse to import a block with a different hash at the given +// / height. +// pub type ForkBlocks = Option, ::Hash)>>; +type ForkBlocks[H runtime.Hash, N runtime.Number] []blockchain.HashNumber[H, N] + +// / Known bad block hashes. +// / +// / This may be used as chain spec extension to filter out known, unwanted forks. +// pub type BadBlocks = Option::Hash>>; +type BadBlocks[H runtime.Hash] map[H]struct{} + // BlockchainEvents is the source of blockchain events. type BlockchainEvents[ H runtime.Hash, diff --git a/internal/client/block_rules.go b/internal/client/block_rules.go index 986639e3e2..0687105190 100644 --- a/internal/client/block_rules.go +++ b/internal/client/block_rules.go @@ -3,7 +3,10 @@ package client -import "github.com/ChainSafe/gossamer/internal/primitives/runtime" +import ( + "github.com/ChainSafe/gossamer/internal/client/api" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" +) // Chain specification rules lookup result. type LookupResult interface { @@ -31,14 +34,14 @@ type BadBlockData[H runtime.Hash, N runtime.Number] struct { } type BlockRules[H runtime.Hash, N runtime.Number] struct { - bad BadBlocks[H] // Set with bad blocks + bad api.BadBlocks[H] // Set with bad blocks forks map[N]H } func NewBlockRules[ H runtime.Hash, N runtime.Number, -](forkBlocks []BadBlockData[H, N], badBlocks BadBlocks[H]) *BlockRules[H, N] { +](forkBlocks api.ForkBlocks[H, N], badBlocks api.BadBlocks[H]) *BlockRules[H, N] { forks := make(map[N]H) for _, block := range forkBlocks { forks[block.Number] = block.Hash diff --git a/internal/client/call_executor.go b/internal/client/call_executor.go new file mode 100644 index 0000000000..d368437f23 --- /dev/null +++ b/internal/client/call_executor.go @@ -0,0 +1,259 @@ +package client + +import ( + "github.com/ChainSafe/gossamer/internal/client/api" + "github.com/ChainSafe/gossamer/internal/client/executor" + "github.com/ChainSafe/gossamer/internal/primitives/core" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" +) + +type executorI interface { + core.CodeExecutor + executor.RuntimeVersionOf +} + +// / Call executor that executes methods locally, querying all required +// / data from local backend. +// pub struct LocalCallExecutor { +type LocalCallExecutor[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + // backend: Arc, + backend api.Backend[H, N, Hasher, Header, E] + // executor: E, + executor executorI + + // code_provider: CodeProvider, + codeProvider CodeProvider[H, N, Hasher, Header, E] + // execution_extensions: Arc>, +} + +// impl LocalCallExecutor +// where +// E: CodeExecutor + RuntimeVersionOf + Clone + 'static, +// B: backend::Backend, +// { +// /// Creates new instance of local call executor. +// pub fn new( +// backend: Arc, +// executor: E, +// client_config: ClientConfig, +// execution_extensions: ExecutionExtensions, +// ) -> sp_blockchain::Result { +// let code_provider = CodeProvider::new(&client_config, executor.clone(), backend.clone())?; + +// Ok(LocalCallExecutor { +// backend, +// executor, +// code_provider, +// execution_extensions: Arc::new(execution_extensions), +// }) +// } +// } + +// impl Clone for LocalCallExecutor +// where +// E: Clone, +// { +// fn clone(&self) -> Self { +// LocalCallExecutor { +// backend: self.backend.clone(), +// executor: self.executor.clone(), +// code_provider: self.code_provider.clone(), +// execution_extensions: self.execution_extensions.clone(), +// } +// } +// } + +// impl CallExecutor for LocalCallExecutor +// where +// B: backend::Backend, +// E: CodeExecutor + RuntimeVersionOf + Clone + 'static, +// Block: BlockT, +// { +// type Error = E::Error; + +// type Backend = B; + +// fn execution_extensions(&self) -> &ExecutionExtensions { +// &self.execution_extensions +// } + +// fn call( +// &self, +// at_hash: Block::Hash, +// method: &str, +// call_data: &[u8], +// context: CallContext, +// ) -> sp_blockchain::Result> { +// let mut changes = OverlayedChanges::default(); +// let at_number = +// self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?; +// let state = self.backend.state_at(at_hash)?; + +// let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); +// let runtime_code = +// state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + +// let runtime_code = self.code_provider.maybe_override_code(runtime_code, &state, at_hash)?.0; + +// let mut extensions = self.execution_extensions.extensions(at_hash, at_number); + +// let mut sm = StateMachine::new( +// &state, +// &mut changes, +// &self.executor, +// method, +// call_data, +// &mut extensions, +// &runtime_code, +// context, +// ) +// .set_parent_hash(at_hash); + +// sm.execute().map_err(Into::into) +// } + +// fn contextual_call( +// &self, +// at_hash: Block::Hash, +// method: &str, +// call_data: &[u8], +// changes: &RefCell>>, +// recorder: &Option>, +// call_context: CallContext, +// extensions: &RefCell, +// ) -> Result, sp_blockchain::Error> { +// let state = self.backend.state_at(at_hash)?; + +// let changes = &mut *changes.borrow_mut(); + +// // It is important to extract the runtime code here before we create the proof +// // recorder to not record it. We also need to fetch the runtime code from `state` to +// // make sure we use the caching layers. +// let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); + +// let runtime_code = +// state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; +// let runtime_code = self.code_provider.maybe_override_code(runtime_code, &state, at_hash)?.0; +// let mut extensions = extensions.borrow_mut(); + +// match recorder { +// Some(recorder) => { +// let trie_state = state.as_trie_backend(); + +// let backend = sp_state_machine::TrieBackendBuilder::wrap(&trie_state) +// .with_recorder(recorder.clone()) +// .build(); + +// let mut state_machine = StateMachine::new( +// &backend, +// changes, +// &self.executor, +// method, +// call_data, +// &mut extensions, +// &runtime_code, +// call_context, +// ) +// .set_parent_hash(at_hash); +// state_machine.execute() +// }, +// None => { +// let mut state_machine = StateMachine::new( +// &state, +// changes, +// &self.executor, +// method, +// call_data, +// &mut extensions, +// &runtime_code, +// call_context, +// ) +// .set_parent_hash(at_hash); +// state_machine.execute() +// }, +// } +// .map_err(Into::into) +// } + +// fn runtime_version(&self, at_hash: Block::Hash) -> sp_blockchain::Result { +// let state = self.backend.state_at(at_hash)?; +// let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); + +// let runtime_code = +// state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; +// self.code_provider +// .maybe_override_code(runtime_code, &state, at_hash) +// .map(|(_, v)| v) +// } + +// fn prove_execution( +// &self, +// at_hash: Block::Hash, +// method: &str, +// call_data: &[u8], +// ) -> sp_blockchain::Result<(Vec, StorageProof)> { +// let at_number = +// self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(at_hash))?; +// let state = self.backend.state_at(at_hash)?; + +// let trie_backend = state.as_trie_backend(); + +// let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(trie_backend); +// let runtime_code = +// state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; +// let runtime_code = self.code_provider.maybe_override_code(runtime_code, &state, at_hash)?.0; + +// sp_state_machine::prove_execution_on_trie_backend( +// trie_backend, +// &mut Default::default(), +// &self.executor, +// method, +// call_data, +// &runtime_code, +// &mut self.execution_extensions.extensions(at_hash, at_number), +// ) +// .map_err(Into::into) +// } +// } + +// impl RuntimeVersionOf for LocalCallExecutor +// where +// E: RuntimeVersionOf, +// Block: BlockT, +// { +// fn runtime_version( +// &self, +// ext: &mut dyn sp_externalities::Externalities, +// runtime_code: &sp_core::traits::RuntimeCode, +// ) -> Result { +// RuntimeVersionOf::runtime_version(&self.executor, ext, runtime_code) +// } +// } + +// impl sp_version::GetRuntimeVersionAt for LocalCallExecutor +// where +// B: backend::Backend, +// E: CodeExecutor + RuntimeVersionOf + Clone + 'static, +// Block: BlockT, +// { +// fn runtime_version(&self, at: Block::Hash) -> Result { +// CallExecutor::runtime_version(self, at).map_err(|e| e.to_string()) +// } +// } + +// impl sp_version::GetNativeVersion for LocalCallExecutor +// where +// B: backend::Backend, +// E: CodeExecutor + sp_version::GetNativeVersion + Clone + 'static, +// Block: BlockT, +// { +// fn native_version(&self) -> &sp_version::NativeVersion { +// self.executor.native_version() +// } +// } diff --git a/internal/client/chain-spec/genesis_block.go b/internal/client/chain-spec/genesis_block.go new file mode 100644 index 0000000000..ccc31ad9d9 --- /dev/null +++ b/internal/client/chain-spec/genesis_block.go @@ -0,0 +1,218 @@ +// Copyright 2025 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package chainspec + +import ( + "errors" + "fmt" + + "github.com/ChainSafe/gossamer/internal/client/api" + "github.com/ChainSafe/gossamer/internal/client/executor" + "github.com/ChainSafe/gossamer/internal/primitives/blockchain" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/runtime/generic" + "github.com/ChainSafe/gossamer/internal/primitives/storage" +) + +var ErrMissingRuntime = errors.New("runtime missing from initial storage, could not read state version") + +func ResolveStateVersionFromWasm[Hasher runtime.Hasher[H], H runtime.Hash]( + storage storage.Storage, + executor executor.RuntimeVersionOf, +) (storage.StateVersion, error) { + panic("unimpl") +} + +// / Create a genesis block, given the initial storage. +// pub fn construct_genesis_block( +// +// state_root: Block::Hash, +// state_version: StateVersion, +// +// ) -> Block { +func ConstructGenesisBlock[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + stateRoot H, + stateVersion storage.StateVersion, +) runtime.Block[H, N, E, Header] { + + // let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + // Vec::new(), + // state_version, + // ); + extrinsicsRoot := (*(new(Hasher))).TrieRoot( + make([]runtime.KeyValue, 0), + stateVersion, + ) + + // Block::new( + // + // <::Header as HeaderT>::new( + // Zero::zero(), + // extrinsics_root, + // state_root, + // Default::default(), + // Default::default(), + // ), + // Default::default(), + // + // ) + + var parentHash H + var digest runtime.Digest + var extrinsics []E + header := generic.NewHeader[N, H, Hasher](0, extrinsicsRoot, stateRoot, parentHash, digest) + block := generic.NewBlock[Hasher, E, N, H, Header]( + any(header).(Header), + extrinsics, + ) + return block +} + +// / Trait for building the genesis block. +// pub trait BuildGenesisBlock { +type BuildGenesisBlock[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] interface { + // /// The import operation used to import the genesis block into the backend. + // type BlockImportOperation; + + // /// Returns the built genesis block along with the block import operation + // /// after setting the genesis storage. + // fn build_genesis_block(self) -> sp_blockchain::Result<(Block, Self::BlockImportOperation)>; + BuildGenesisBlock(runtime.Block[H, N, E, Header], api.BlockImportOperation[N, H, Hasher, Header, E]) +} + +// /// Default genesis block builder in Substrate. +// pub struct GenesisBlockBuilder { +type GenesisBlockBuilder[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + // genesis_storage: Storage, + gensisStorage storage.Storage + // commit_genesis_state: bool, + commitGenesisState bool + // backend: Arc, + backend api.Backend[H, N, Hasher, Header, E] + // executor: E, + executor executor.RuntimeVersionOf + // _phantom: PhantomData, +} + +// impl, E: RuntimeVersionOf> GenesisBlockBuilder { +// /// Constructs a new instance of [`GenesisBlockBuilder`]. +// pub fn new( +// build_genesis_storage: &dyn BuildStorage, +// commit_genesis_state: bool, +// backend: Arc, +// executor: E, +// ) -> sp_blockchain::Result { +// let genesis_storage = +// build_genesis_storage.build_storage().map_err(sp_blockchain::Error::Storage)?; +// Self::new_with_storage(genesis_storage, commit_genesis_state, backend, executor) +// } +func NewGenesisBlockBuilder[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + buildGenesisStorage runtime.BuildStorage, + commitGenesisState bool, + backend api.Backend[H, N, Hasher, Header, E], + executor executor.RuntimeVersionOf, +) (*GenesisBlockBuilder[H, N, Hasher, Header, E], error) { + genesisStorage, err := buildGenesisStorage.BuildStorage() + if err != nil { + return nil, fmt.Errorf("%w: %s", blockchain.ErrStorage, err) + } + return NewGenesisBlockBuilderWithStorage( + genesisStorage, + commitGenesisState, + backend, + executor, + ), nil +} + +// /// Constructs a new instance of [`GenesisBlockBuilder`] using provided storage. +// pub fn new_with_storage( +// genesis_storage: Storage, +// commit_genesis_state: bool, +// backend: Arc, +// executor: E, +// ) -> sp_blockchain::Result { +// Ok(Self { +// genesis_storage, +// commit_genesis_state, +// backend, +// executor, +// _phantom: PhantomData::, +// }) +// } +// } +func NewGenesisBlockBuilderWithStorage[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + genesisStorage storage.Storage, + commitGenesisState bool, + backend api.Backend[H, N, Hasher, Header, E], + executor executor.RuntimeVersionOf, +) *GenesisBlockBuilder[H, N, Hasher, Header, E] { + return &GenesisBlockBuilder[H, N, Hasher, Header, E]{ + gensisStorage: genesisStorage, + commitGenesisState: commitGenesisState, + backend: backend, + executor: executor, + } +} + +// impl, E: RuntimeVersionOf> BuildGenesisBlock +// for GenesisBlockBuilder +// { +// type BlockImportOperation = >::BlockImportOperation; + +// fn build_genesis_block(self) -> sp_blockchain::Result<(Block, Self::BlockImportOperation)> { +func (g *GenesisBlockBuilder[H, N, Hasher, Header, E]) BuildGenesisBlock() (runtime.Block[H, N, E, Header], api.BlockImportOperation[N, H, Hasher, Header, E], error) { + // let Self { genesis_storage, commit_genesis_state, backend, executor, _phantom } = self; + + // let genesis_state_version = + // resolve_state_version_from_wasm::<_, HashingFor>(&genesis_storage, &executor)?; + // let mut op = backend.begin_operation()?; + // let state_root = + // op.set_genesis_state(genesis_storage, commit_genesis_state, genesis_state_version)?; + // let genesis_block = construct_genesis_block::(state_root, genesis_state_version); + + genesisStateVersion, err := ResolveStateVersionFromWasm[Hasher, H](g.gensisStorage, g.executor) + if err != nil { + return nil, nil, err + } + op, err := g.backend.BeginOperation() + stateRoot, err := op.SetGenesisState(g.gensisStorage, g.commitGenesisState, genesisStateVersion) + if err != nil { + return nil, nil, err + } + genesisBlock := ConstructGenesisBlock[H, N, Hasher, Header, E](stateRoot, genesisStateVersion) + // Ok((genesis_block, op)) + return genesisBlock, op, nil +} + +// } diff --git a/internal/client/client.go b/internal/client/client.go index 731c82687d..c7ee81b595 100644 --- a/internal/client/client.go +++ b/internal/client/client.go @@ -11,13 +11,14 @@ import ( "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/internal/client/api" + chainspec "github.com/ChainSafe/gossamer/internal/client/chain-spec" "github.com/ChainSafe/gossamer/internal/client/consensus/common" "github.com/ChainSafe/gossamer/internal/client/executor" genesisblock "github.com/ChainSafe/gossamer/internal/client/genesis-block" "github.com/ChainSafe/gossamer/internal/log" primitives_api "github.com/ChainSafe/gossamer/internal/primitives/api" "github.com/ChainSafe/gossamer/internal/primitives/blockchain" - primivite_consensus_common "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" + primitives_common "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" "github.com/ChainSafe/gossamer/internal/primitives/core" "github.com/ChainSafe/gossamer/internal/primitives/runtime" "github.com/ChainSafe/gossamer/internal/primitives/runtime/generic" @@ -28,8 +29,6 @@ import ( var logger = log.NewFromGlobal(log.AddContext("pkg", "client")) -type BadBlocks[H runtime.Hash] map[H]struct{} - type prepareStorageChangesResult interface { isPrepareStorageChangesResult() } @@ -83,15 +82,23 @@ type ClientConfig[N runtime.Number] struct { EnableImportProofRecording bool } +func NewClientConfig[N runtime.Number]() ClientConfig[N] { + return ClientConfig[N]{ + OffchainWorkerEnabled: false, + OffchainIndexingAPI: false, + WasmRuntimeSubstitutes: make(map[N][]byte), + NoGenesis: false, + EnableImportProofRecording: false, + } +} + // Client type that implements a number of client interfaces type Client[ H runtime.Hash, Hasher runtime.Hasher[H], N runtime.Number, E runtime.Extrinsic, - Executor ExecutorT, Header runtime.Header[N, H], - RA primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[N, E, H, Hasher, statemachine.Backend[H, Hasher], any]], ] struct { backend api.Backend[H, N, Hasher, Header, E] executor Executor @@ -114,10 +121,11 @@ type Client[ unpinWorkerChan chan<- api.UnpinWorkerMessage[H] blockRules BlockRules[H, N] config ClientConfig[N] - runtimeConstructor RA + runtimeConstructor primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[N, E, H, Hasher, statemachine.Backend[H, Hasher], any, Header]] } -type ExecutorT interface { +type Executor interface { + api.CallExecutor core.CodeExecutor executor.RuntimeVersionOf } @@ -128,20 +136,63 @@ func New[ Hasher runtime.Hasher[H], N runtime.Number, E runtime.Extrinsic, - Executor ExecutorT, Header runtime.Header[N, H], - RA primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[N, E, H, Hasher, statemachine.Backend[H, Hasher], any]], ]( backend api.Backend[H, N, Hasher, Header, E], config ClientConfig[N], executor Executor, - runtimeConstructor RA, -) *Client[H, Hasher, N, E, Executor, Header, RA] { + runtimeConstructor primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[N, E, H, Hasher, statemachine.Backend[H, Hasher], any, Header]], + genesisBlockBuilder *chainspec.GenesisBlockBuilder[H, N, Hasher, Header, E], + forkBlocks api.ForkBlocks[H, N], + badBlocks api.BadBlocks[H], +) (*Client[H, Hasher, N, E, Header], error) { + // let info = backend.blockchain().info(); + info := backend.Blockchain().Info() + // if info.finalized_state.is_none() { + if info.FinalizedState == nil && genesisBlockBuilder != nil { + // let (genesis_block, mut op) = genesis_block_builder.build_genesis_block()?; + genesisBlock, op, err := genesisBlockBuilder.BuildGenesisBlock() + if err != nil { + return nil, err + } + // info!( + // "🔨 Initializing Genesis block/state (state: {}, header-hash: {})", + // genesis_block.header().state_root(), + // genesis_block.header().hash() + // ); + logger.Infof( + "🔨 Initializing Genesis block/state (state: %s, header-hash: %s)", + genesisBlock.Header().StateRoot(), genesisBlock.Header().Hash(), + ) + // Genesis may be written after some blocks have been imported and finalized. + // So we only finalize it when the database is empty. + // let block_state = if info.best_hash == Default::default() { + // NewBlockState::Final + // } else { + // NewBlockState::Normal + // }; + var blockState api.NewBlockState + if info.BestHash == *(new(H)) { + blockState = api.NewBlockStateFinal + } else { + blockState = api.NewBlockStateNormal + } + // let (header, body) = genesis_block.deconstruct(); + header, body := genesisBlock.Deconstruct() + // op.set_block_data(header, Some(body), None, None, block_state)?; + op.SetBlockData(header, body, nil, nil, blockState) + // backend.commit_operation(op)?; + err = backend.CommitOperation(op) + if err != nil { + return nil, err + } + } + unpinWorkerChan := make(chan api.UnpinWorkerMessage[H]) npw := newNotificationPinningWorker(unpinWorkerChan, backend) go npw.run() - return &Client[H, Hasher, N, E, Executor, Header, RA]{ + return &Client[H, Hasher, N, E, Header]{ backend: backend, executor: executor, storageNotifications: api.NewStorageNotifications[H](), @@ -151,10 +202,11 @@ func New[ unpinWorkerChan: unpinWorkerChan, config: config, runtimeConstructor: runtimeConstructor, - } + blockRules: *NewBlockRules[H, N](forkBlocks, badBlocks), + }, nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) announcePin(message api.AnnouncePin[H]) error { +func (c *Client[H, Hasher, N, E, Header]) announcePin(message api.AnnouncePin[H]) error { select { case c.unpinWorkerChan <- message: return nil @@ -163,7 +215,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) announcePin(message api. } } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) unpin(message api.Unpin[H]) error { +func (c *Client[H, Hasher, N, E, Header]) unpin(message api.Unpin[H]) error { select { case c.unpinWorkerChan <- message: return nil @@ -172,7 +224,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) unpin(message api.Unpin[ } } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) lockImportRun( +func (c *Client[H, Hasher, N, E, Header]) lockImportRun( f func(*api.ClientImportOperation[H, Hasher, N, Header, E]) (any, error), ) (any, error) { c.backend.GetImportLock().Lock() @@ -277,7 +329,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) lockImportRun( return result, nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) LockImportRun( +func (c *Client[H, Hasher, N, E, Header]) LockImportRun( f func(*api.ClientImportOperation[H, Hasher, N, Header, E]) (any, error), ) (any, error) { result, err := c.lockImportRun(f) @@ -290,7 +342,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) LockImportRun( const notifyFinalizedTimeout = 5 * time.Second const notifyBlockImportTimeout = notifyFinalizedTimeout -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) notifyFinalized( +func (c *Client[H, Hasher, N, E, Header]) notifyFinalized( notification *api.FinalityNotification[H, N, Header], ) error { c.finalityNotificationChansMtx.Lock() @@ -356,7 +408,7 @@ func notifyChans[M any](msg M, chans map[chan M]any, timeout time.Duration) { } } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) notifyImported( +func (c *Client[H, Hasher, N, E, Header]) notifyImported( notification *api.BlockImportNotification[H, N, Header], importNotificationAction api.ImportNotificationAction, storageChanges *api.StorageChanges, @@ -426,19 +478,19 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) notifyImported( return nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RegisterImportAction(op api.OnImportAction[H, N, Header]) { +func (c *Client[H, Hasher, N, E, Header]) RegisterImportAction(op api.OnImportAction[H, N, Header]) { c.importActionsMtx.Lock() defer c.importActionsMtx.Unlock() c.importActions = append(c.importActions, op) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RegisterFinalityAction(op api.OnFinalityAction[H, N, Header]) { +func (c *Client[H, Hasher, N, E, Header]) RegisterFinalityAction(op api.OnFinalityAction[H, N, Header]) { c.finalityActionsMtx.Lock() defer c.finalityActionsMtx.Unlock() c.finalityActions = append(c.finalityActions, op) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RegisterImportNotificationStream() api.ImportNotifications[ +func (c *Client[H, Hasher, N, E, Header]) RegisterImportNotificationStream() api.ImportNotifications[ H, N, Header] { ch := make(chan api.BlockImportNotification[H, N, Header]) c.importNotificationChansMtx.Lock() @@ -447,7 +499,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RegisterImportNotificati return ch } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) UnregisterImportNotificationStream( +func (c *Client[H, Hasher, N, E, Header]) UnregisterImportNotificationStream( ch api.ImportNotifications[H, N, Header], ) { c.importNotificationChansMtx.Lock() @@ -459,7 +511,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) UnregisterImportNotifica delete(c.importNotificationChans, ch) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RegisterEveryImportNotificationStream() api.ImportNotifications[ +func (c *Client[H, Hasher, N, E, Header]) RegisterEveryImportNotificationStream() api.ImportNotifications[ H, N, Header] { ch := make(chan api.BlockImportNotification[H, N, Header]) c.everyImportNotificationChansMtx.Lock() @@ -468,7 +520,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RegisterEveryImportNotif return ch } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) UnregisterEveryImportNotificationStream( +func (c *Client[H, Hasher, N, E, Header]) UnregisterEveryImportNotificationStream( ch api.ImportNotifications[H, N, Header], ) { c.everyImportNotificationChansMtx.Lock() @@ -480,7 +532,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) UnregisterEveryImportNot delete(c.everyImportNotificationChans, ch) } -func (c *Client[H, _, N, E, Executor, Header, RA]) RegisterFinalityNotificationStream() api.FinalityNotifications[ +func (c *Client[H, Hasher, N, E, Header]) RegisterFinalityNotificationStream() api.FinalityNotifications[ H, N, Header] { ch := make(chan api.FinalityNotification[H, N, Header]) c.finalityNotificationChansMtx.Lock() @@ -489,7 +541,7 @@ func (c *Client[H, _, N, E, Executor, Header, RA]) RegisterFinalityNotificationS return ch } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) UnregisterFinalityNotificationStream( +func (c *Client[H, Hasher, N, E, Header]) UnregisterFinalityNotificationStream( ch api.FinalityNotifications[H, N, Header], ) { c.finalityNotificationChansMtx.Lock() @@ -501,14 +553,14 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) UnregisterFinalityNotifi delete(c.finalityNotificationChans, ch) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) StorageChangesNotificationStream( +func (c *Client[H, Hasher, N, E, Header]) StorageChangesNotificationStream( filterKeys []storage.StorageKey, childFilterKeys []api.ChildFilterKeys, ) api.StorageEventStream[H] { return c.storageNotifications.Listen(filterKeys, childFilterKeys) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) CompareAndSetBlockData(bd *types.BlockData) error { +func (c *Client[H, Hasher, N, E, Header]) CompareAndSetBlockData(bd *types.BlockData) error { storage := c.backend.OffchainStorage() hash := bd.Hash[:] @@ -527,45 +579,45 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) CompareAndSetBlockData(b // HeaderBackend implementation for Client -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Header(hash H) (*Header, error) { +func (c *Client[H, Hasher, N, E, Header]) Header(hash H) (*Header, error) { return c.backend.Blockchain().Header(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Body(hash H) ([]E, error) { +func (c *Client[H, Hasher, N, E, Header]) Body(hash H) ([]E, error) { return c.backend.Blockchain().Body(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Info() blockchain.Info[H, N] { +func (c *Client[H, Hasher, N, E, Header]) Info() blockchain.Info[H, N] { return c.backend.Blockchain().Info() } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Status(hash H) (blockchain.BlockStatus, error) { +func (c *Client[H, Hasher, N, E, Header]) Status(hash H) (blockchain.BlockStatus, error) { return c.backend.Blockchain().Status(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Number(hash H) (*N, error) { +func (c *Client[H, Hasher, N, E, Header]) Number(hash H) (*N, error) { return c.backend.Blockchain().Number(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Hash(number N) (*H, error) { +func (c *Client[H, Hasher, N, E, Header]) Hash(number N) (*H, error) { return c.backend.Blockchain().Hash(number) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) BlockHashFromID(id generic.BlockID) (*H, error) { +func (c *Client[H, Hasher, N, E, Header]) BlockHashFromID(id generic.BlockID) (*H, error) { return c.backend.Blockchain().BlockHashFromID(id) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) BlockNumberFromID(id generic.BlockID) (*N, error) { +func (c *Client[H, Hasher, N, E, Header]) BlockNumberFromID(id generic.BlockID) (*N, error) { return c.backend.Blockchain().BlockNumberFromID(id) } // BlockBackend implementation for Client -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) BlockBody(hash H) ([]E, error) { +func (c *Client[H, Hasher, N, E, Header]) BlockBody(hash H) ([]E, error) { return c.backend.Blockchain().Body(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Block(hash H) (*generic.SignedBlock[N, H, Hasher, E], error) { +func (c *Client[H, Hasher, N, E, Header]) Block(hash H) (*generic.SignedBlock[N, H, Hasher, E, Header], error) { header, err := c.Header(hash) if err != nil { return nil, err @@ -590,61 +642,61 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Block(hash H) (*generic. return nil, nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) BlockStatus(hash H) ( - primivite_consensus_common.BlockStatus, error, +func (c *Client[H, Hasher, N, E, Header]) BlockStatus(hash H) ( + primitives_common.BlockStatus, error, ) { c.importingBlockMtx.RLock() if c.importingBlock != nil && *c.importingBlock == hash { - return primivite_consensus_common.BlockStatusQueued, nil + return primitives_common.BlockStatusQueued, nil } c.importingBlockMtx.RUnlock() number, err := c.backend.Blockchain().Number(hash) if err != nil { - return primivite_consensus_common.BlockStatusUnknown, err + return primitives_common.BlockStatusUnknown, err } if number == nil { - return primivite_consensus_common.BlockStatusUnknown, nil + return primitives_common.BlockStatusUnknown, nil } if c.backend.HaveStateAt(hash, *number) { - return primivite_consensus_common.BlockStatusInChainWithState, nil + return primitives_common.BlockStatusInChainWithState, nil } else { - return primivite_consensus_common.BlockStatusInChainPruned, nil + return primitives_common.BlockStatusInChainPruned, nil } } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Justifications(hash H) (runtime.Justifications, error) { +func (c *Client[H, Hasher, N, E, Header]) Justifications(hash H) (runtime.Justifications, error) { return c.backend.Blockchain().Justifications(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) BlockHash(number N) (*H, error) { +func (c *Client[H, Hasher, N, E, Header]) BlockHash(number N) (*H, error) { return c.backend.Blockchain().Hash(number) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) IndexedTransaction(hash H) ([]byte, error) { +func (c *Client[H, Hasher, N, E, Header]) IndexedTransaction(hash H) ([]byte, error) { return c.backend.Blockchain().IndexedTransaction(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) HasIndexedTransaction(hash H) (bool, error) { +func (c *Client[H, Hasher, N, E, Header]) HasIndexedTransaction(hash H) (bool, error) { return c.backend.Blockchain().HasIndexedTransaction(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) BlockIndexedBody(hash H) ([][]byte, error) { +func (c *Client[H, Hasher, N, E, Header]) BlockIndexedBody(hash H) ([][]byte, error) { return c.backend.Blockchain().BlockIndexedBody(hash) } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RequiresFullSync() bool { +func (c *Client[H, Hasher, N, E, Header]) RequiresFullSync() bool { return c.backend.RequiresFullSync() } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) Children(parent H) ([]H, error) { +func (c *Client[H, Hasher, N, E, Header]) Children(parent H) ([]H, error) { return c.backend.Blockchain().Children(parent) } // Note: This is an async function so the plan is to ensure we do call it in a goroutine -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) CheckBlock(block common.BlockCheckParams[H, N]) ( +func (c *Client[H, Hasher, N, E, Header]) CheckBlock(block common.BlockCheckParams[H, N]) ( common.ImportResult, error, ) { // Check the block against white and black lists if any are defined @@ -673,13 +725,13 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) CheckBlock(block common. } switch blockStatus { - case primivite_consensus_common.BlockStatusInChainWithState, primivite_consensus_common.BlockStatusQueued: + case primitives_common.BlockStatusInChainWithState, primitives_common.BlockStatusQueued: return common.ImportResultAlreadyInChain{}, nil - case primivite_consensus_common.BlockStatusInChainPruned: + case primitives_common.BlockStatusInChainPruned: if !block.ImportExisting { return common.ImportResultAlreadyInChain{}, nil } - case primivite_consensus_common.BlockStatusUnknown: + case primitives_common.BlockStatusUnknown: // do nothing default: panic("unreachable") @@ -691,13 +743,13 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) CheckBlock(block common. } switch parentStatus { - case primivite_consensus_common.BlockStatusInChainWithState, primivite_consensus_common.BlockStatusQueued: + case primitives_common.BlockStatusInChainWithState, primitives_common.BlockStatusQueued: // do nothing - case primivite_consensus_common.BlockStatusUnknown: + case primitives_common.BlockStatusUnknown: if !block.AllowMissingParent { return common.ImportResultUnknownParent{}, nil } - case primivite_consensus_common.BlockStatusInChainPruned: + case primitives_common.BlockStatusInChainPruned: if !block.AllowMissingParent { return common.ImportResultMissingState{}, nil } @@ -711,7 +763,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) CheckBlock(block common. } // Note: This is an async function so the plan is to ensure we do call it in a goroutine -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) ImportBlock( +func (c *Client[H, Hasher, N, E, Header]) ImportBlock( block *common.BlockImportParams[H, N, E, Header], ) (common.ImportResult, error) { prepareStorageResult, err := c.prepareBlockStorageChanges(block) @@ -742,7 +794,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) ImportBlock( return importResult.(common.ImportResult), nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) prepareBlockStorageChanges( +func (c *Client[H, Hasher, N, E, Header]) prepareBlockStorageChanges( importBlock *common.BlockImportParams[H, N, E, Header], ) (prepareStorageChangesResult, error) { parentHash := importBlock.Header.ParentHash() @@ -757,7 +809,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) prepareBlockStorageChang return nil, err } - if status == primivite_consensus_common.BlockStatusInChainPruned { + if status == primitives_common.BlockStatusInChainPruned { switch action := stateAction.(type) { case common.StateActionApplyChanges: if _, ok := action.StorageChanges.(common.Changes[H, Hasher]); ok { @@ -772,7 +824,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) prepareBlockStorageChang } else if action, ok := stateAction.(common.StateActionApplyChanges); ok { enactState = true storageChanges = action.StorageChanges - } else if status == primivite_consensus_common.BlockStatusUnknown { + } else if status == primitives_common.BlockStatusUnknown { return prepareStorageChangesResultDiscard{common.ImportResultUnknownParent{}}, nil } else if _, ok := stateAction.(common.StateActionSkip); ok { enactState = false @@ -792,7 +844,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) prepareBlockStorageChang storageChangesToApply = storageChanges } else if enactState && storageChanges == nil && importBlock.Body != nil { // We should enact state, but don't have any storage changes, so we need to execute the block - runtimeApi := c.RuntimeApi() + runtimeApi := c.RuntimeAPI() runtimeApi.SetCallContext(core.CallContextOnchain) if c.config.EnableImportProofRecording { @@ -836,7 +888,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) prepareBlockStorageChang return prepareStorageChangesResultImport{storageChangesToApply}, nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) applyBlock( +func (c *Client[H, Hasher, N, E, Header]) applyBlock( operation *api.ClientImportOperation[H, Hasher, N, Header, E], importBlock common.BlockImportParams[H, N, E, Header], storageChanges common.StorageChanges, @@ -894,9 +946,9 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) applyBlock( } //gocyclo:ignore -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) executeAndImportBlock( +func (c *Client[H, Hasher, N, E, Header]) executeAndImportBlock( operation *api.ClientImportOperation[H, Hasher, N, Header, E], - origin primivite_consensus_common.BlockOrigin, + origin primitives_common.BlockOrigin, hash H, importHeaders PrePostHeaders[N, H, Header], justifications runtime.Justifications, @@ -937,9 +989,9 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) executeAndImportBlock( // this is a fairly arbitrary choice of where to draw the line on making notifications, // but the general goal is to only make notifications when we are already fully synced // and get a new chain head. - makeNotifications := origin == primivite_consensus_common.NetworkBroadcastBlockOrigin || - origin == primivite_consensus_common.OwnBlockOrigin || - origin == primivite_consensus_common.ConsensusBroadcastBlockOrigin + makeNotifications := origin == primitives_common.NetworkBroadcastBlockOrigin || + origin == primitives_common.OwnBlockOrigin || + origin == primitives_common.ConsensusBroadcastBlockOrigin var finalStorageChanges *api.StorageChanges @@ -1043,9 +1095,9 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) executeAndImportBlock( if !gapBlock && finalized { switch fc := forkchoice.(type) { - case common.LongestChain: + case common.ForkChoiceStrategyLongestChain: isNewBest = importHeaders.Post().Number() > info.BestNumber - case common.Custom: + case common.ForkChoiceStrategyCustom: isNewBest = bool(fc) default: panic("unreachable") @@ -1173,7 +1225,7 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) executeAndImportBlock( return common.ImportResultImported{IsNewBest: isNewBest}, nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) applyFinalityWithBlockHash( +func (c *Client[H, Hasher, N, E, Header]) applyFinalityWithBlockHash( operation *api.ClientImportOperation[H, Hasher, N, Header, E], hash H, justification *runtime.Justification, @@ -1297,10 +1349,11 @@ func (c *Client[H, Hasher, N, E, Executor, Header, RA]) applyFinalityWithBlockHa return nil } -func (c *Client[H, Hasher, N, E, Executor, Header, RA]) RuntimeApi() primitives_api.ApiExt[ +func (c *Client[H, Hasher, N, E, Header]) RuntimeAPI() primitives_api.ApiExt[ N, E, H, Hasher, statemachine.Backend[H, Hasher], any, + Header, ] { - return c.runtimeConstructor.ConstructRuntimeApi() + return c.runtimeConstructor.ConstructRuntimeAPI() } diff --git a/internal/client/client_test.go b/internal/client/client_test.go index da54c15d3f..a461f807d9 100644 --- a/internal/client/client_test.go +++ b/internal/client/client_test.go @@ -16,7 +16,7 @@ import ( memorykvdb "github.com/ChainSafe/gossamer/internal/kvdb/memory-kvdb" primitives_api "github.com/ChainSafe/gossamer/internal/primitives/api" "github.com/ChainSafe/gossamer/internal/primitives/blockchain" - primivite_consensus_common "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" + primitives_common "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" "github.com/ChainSafe/gossamer/internal/primitives/core" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" "github.com/ChainSafe/gossamer/internal/primitives/database" @@ -36,12 +36,7 @@ type TestClient struct { runtime.BlakeTwo256, uint64, runtime.OpaqueExtrinsic, - ExecutorT, *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], - primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[ - uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, - statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, - ]], ] } @@ -63,9 +58,10 @@ var ( runtime.OpaqueExtrinsic, *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], ] = &TestClient{} - _ primitives_api.ProvideRuntimeApi[primitives_api.ApiExt[ + _ primitives_api.ProvideRuntimeAPI[primitives_api.ApiExt[ uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, + *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], ]] = &TestClient{} ) @@ -88,20 +84,21 @@ func (e *TestExecutor) RuntimeVersion( panic("not implemented") } -func NewTestExecutor(t *testing.T) ExecutorT { +func NewTestExecutor(t *testing.T) Executor { t.Helper() return &TestExecutor{} } type RuntimeConstructor struct{} -func (e *RuntimeConstructor) ConstructRuntimeApi() primitives_api.ApiExt[ +func (e *RuntimeConstructor) ConstructRuntimeAPI() primitives_api.ApiExt[ uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, + *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], ] { panic("not implemented") } @@ -109,6 +106,7 @@ func (e *RuntimeConstructor) ConstructRuntimeApi() primitives_api.ApiExt[ func NewRuntimeConstructor(t *testing.T) primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[ uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, + *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], ]] { t.Helper() return &RuntimeConstructor{} @@ -163,19 +161,19 @@ func newTestClient(t *testing.T) *Client[ runtime.BlakeTwo256, uint64, runtime.OpaqueExtrinsic, - ExecutorT, *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], - primitives_api.ConstructRuntimeApi[primitives_api.ApiExt[ - uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, - statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, - ]], ] { - return New( + client, err := New( NewTestBackend(t, db.BlocksPruningKeepFinalized{}, 0), ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), + nil, + nil, + nil, ) + require.NoError(t, err) + return client } func TestNew(t *testing.T) { @@ -470,7 +468,8 @@ func TestHeaderBackendImplementation(t *testing.T) { backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) // Get Header header, err := c.Header(expectedHash) @@ -525,7 +524,11 @@ func TestBlockBackendImplementation(t *testing.T) { blockchainMock := mocks.NewBlockchainBackend[hash.H256, uint64, *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], runtime.OpaqueExtrinsic](t) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + backendMock.EXPECT().Blockchain().Return(blockchainMock) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) + + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) expectedHeader := generic.NewHeader[uint64, hash.H256, runtime.BlakeTwo256]( 1, @@ -544,7 +547,7 @@ func TestBlockBackendImplementation(t *testing.T) { expectedExtrinsics := []runtime.OpaqueExtrinsic{} // skipcq: GO-W1027 blockchainMock.EXPECT().Body(expectedHash).Return(expectedExtrinsics, nil) - expectedStatus := primivite_consensus_common.BlockStatusInChainWithState + expectedStatus := primitives_common.BlockStatusInChainWithState expectedIndexedExtrinsics := [][]byte{ []byte("extrinsic1"), @@ -676,11 +679,13 @@ func TestCheckBlock(t *testing.T) { *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], runtime.OpaqueExtrinsic](t) blockchainMock.EXPECT().Number(importedBlockWithStatus.Hash).Return(&importedBlockWithStatus.Number, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().Blockchain().Return(blockchainMock) backendMock.EXPECT().HaveStateAt(importedBlockWithStatus.Hash, importedBlockWithStatus.Number).Return(true) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.CheckBlock(importedBlockWithStatus) require.NoError(t, err) require.Equal(t, common.ImportResultAlreadyInChain{}, result) @@ -701,11 +706,13 @@ func TestCheckBlock(t *testing.T) { *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], runtime.OpaqueExtrinsic](t) blockchainMock.EXPECT().Number(prunedBlock.Hash).Return(&prunedBlock.Number, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().Blockchain().Return(blockchainMock) backendMock.EXPECT().HaveStateAt(prunedBlock.Hash, prunedBlock.Number).Return(false) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.CheckBlock(prunedBlock) require.NoError(t, err) @@ -728,10 +735,12 @@ func TestCheckBlock(t *testing.T) { blockchainMock.EXPECT().Number(blockUnknownParent.Hash).Return(nil, nil) blockchainMock.EXPECT().Number(blockUnknownParent.ParentHash).Return(nil, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.CheckBlock(blockUnknownParent) require.NoError(t, err) @@ -758,12 +767,14 @@ func TestCheckBlock(t *testing.T) { parentNumber := blockUnknownParent.Number - 1 blockchainMock.EXPECT().Number(parentHash).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(parentHash, parentNumber).Return(false) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.CheckBlock(blockUnknownParent) require.NoError(t, err) @@ -785,6 +796,7 @@ func TestCheckBlock(t *testing.T) { *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], runtime.OpaqueExtrinsic](t) blockchainMock.EXPECT().Number(blockUnknownParent.Hash).Return(nil, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) parentHash := blockUnknownParent.ParentHash parentNumber := blockUnknownParent.Number - 1 @@ -793,7 +805,8 @@ func TestCheckBlock(t *testing.T) { backendMock.EXPECT().HaveStateAt(parentHash, parentNumber).Return(true) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.CheckBlock(blockUnknownParent) require.NoError(t, err) @@ -823,12 +836,14 @@ func TestPrepareBlockStorageChanges(t *testing.T) { expectedError := errors.New("error") blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(nil, expectedError) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) - _, err := c.prepareBlockStorageChanges(&block) + _, err = c.prepareBlockStorageChanges(&block) require.Error(t, err) require.Equal(t, expectedError, err) @@ -859,11 +874,14 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) + backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(false) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -892,9 +910,11 @@ func TestPrepareBlockStorageChanges(t *testing.T) { *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], runtime.OpaqueExtrinsic](t) blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(nil, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -924,10 +944,12 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(false) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -957,10 +979,12 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(false) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -994,10 +1018,12 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(true) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -1027,10 +1053,12 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(true) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -1060,10 +1088,12 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(true) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -1093,10 +1123,12 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(true) backendMock.EXPECT().Blockchain().Return(blockchainMock) - c := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t)) + c, err := New(backendMock, ClientConfig[uint64]{}, NewTestExecutor(t), NewRuntimeConstructor(t), nil, nil, nil) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) @@ -1131,21 +1163,24 @@ func TestPrepareBlockStorageChanges(t *testing.T) { parentNumber := block.Header.Number() - 1 blockchainMock.EXPECT().Number(block.Header.ParentHash()).Return(&parentNumber, nil) + blockchainMock.EXPECT().Info().Return(blockchain.Info[hash.H256, uint64]{}) backendMock.EXPECT().HaveStateAt(block.Header.ParentHash(), parentNumber).Return(true) backendMock.EXPECT().Blockchain().Return(blockchainMock) runtimeConstructorMock := mocks.NewConstructRuntimeApi[ - uint64, runtime.OpaqueExtrinsic, hash.H256, - runtime.BlakeTwo256, statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, - primitives_api.ApiExt[ - uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, - statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, - ]](t) + // uint64, runtime.OpaqueExtrinsic, hash.H256, + // runtime.BlakeTwo256, statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, + primitives_api.ApiExt[ + uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, + statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, + *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], + ]](t) runtimeApi := mocks.NewApiExt[ uint64, runtime.OpaqueExtrinsic, hash.H256, runtime.BlakeTwo256, statemachine.Backend[hash.H256, runtime.BlakeTwo256], any, + *generic.Header[uint64, hash.H256, runtime.BlakeTwo256], ](t) runtimeApi.EXPECT().RecordProof().Return() @@ -1165,13 +1200,16 @@ func TestPrepareBlockStorageChanges(t *testing.T) { runtimeApi.EXPECT().IntoStorageChanges(state, block.Header.ParentHash()).Return(storageChanges, nil) - runtimeConstructorMock.EXPECT().ConstructRuntimeApi().Return(runtimeApi) + runtimeConstructorMock.EXPECT().ConstructRuntimeAPI().Return(runtimeApi) - c := New(backendMock, + c, err := New(backendMock, ClientConfig[uint64]{EnableImportProofRecording: true}, NewTestExecutor(t), runtimeConstructorMock, + nil, + nil, nil, ) + require.NoError(t, err) result, err := c.prepareBlockStorageChanges(&block) require.NoError(t, err) diff --git a/internal/client/code_provider.go b/internal/client/code_provider.go new file mode 100644 index 0000000000..86acf0b66f --- /dev/null +++ b/internal/client/code_provider.go @@ -0,0 +1,155 @@ +package client + +import ( + "github.com/ChainSafe/gossamer/internal/client/api" + "github.com/ChainSafe/gossamer/internal/client/executor" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" +) + +// / Provider for fetching `:code` of a block. +// / +// / As a node can run with code overrides or substitutes, this will ensure that these are taken into +// / account before returning the actual `code` for a block. +// pub struct CodeProvider { +type CodeProvider[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + // backend: Arc, + backend api.Backend[H, N, Hasher, Header, E] + // executor: Arc, + executor executor.RuntimeVersionOf + // wasm_override: Arc>, + // wasm_substitutes: WasmSubstitutes, +} + +// impl Clone for CodeProvider { +// fn clone(&self) -> Self { +// Self { +// backend: self.backend.clone(), +// executor: self.executor.clone(), +// wasm_override: self.wasm_override.clone(), +// wasm_substitutes: self.wasm_substitutes.clone(), +// } +// } +// } + +// impl CodeProvider +// where +// Block: BlockT, +// Backend: backend::Backend, +// Executor: RuntimeVersionOf, +// { +// /// Create a new instance. +// pub fn new( +// client_config: &ClientConfig, +// executor: Executor, +// backend: Arc, +// ) -> sp_blockchain::Result { +// let wasm_override = client_config +// .wasm_runtime_overrides +// .as_ref() +// .map(|p| WasmOverride::new(p.clone(), &executor)) +// .transpose()?; + +// let executor = Arc::new(executor); + +// let wasm_substitutes = WasmSubstitutes::new( +// client_config.wasm_runtime_substitutes.clone(), +// executor.clone(), +// backend.clone(), +// )?; + +// Ok(Self { backend, executor, wasm_override: Arc::new(wasm_override), wasm_substitutes }) +// } + +// /// Returns the `:code` for the given `block`. +// /// +// /// This takes into account potential overrides/substitutes. +// pub fn code_at_ignoring_overrides(&self, block: Block::Hash) -> sp_blockchain::Result> { +// let state = self.backend.state_at(block)?; + +// let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); +// let runtime_code = +// state_runtime_code.runtime_code().map_err(sp_blockchain::Error::RuntimeCode)?; + +// self.maybe_override_code_internal(runtime_code, &state, block, true) +// .and_then(|r| { +// r.0.fetch_runtime_code().map(Into::into).ok_or_else(|| { +// sp_blockchain::Error::Backend("Could not find `:code` in backend.".into()) +// }) +// }) +// } + +// /// Maybe override the given `onchain_code`. +// /// +// /// This takes into account potential overrides/substitutes. +// pub fn maybe_override_code<'a>( +// &'a self, +// onchain_code: RuntimeCode<'a>, +// state: &Backend::State, +// hash: Block::Hash, +// ) -> sp_blockchain::Result<(RuntimeCode<'a>, RuntimeVersion)> { +// self.maybe_override_code_internal(onchain_code, state, hash, false) +// } + +// /// Maybe override the given `onchain_code`. +// /// +// /// This takes into account potential overrides(depending on `ignore_overrides`)/substitutes. +// fn maybe_override_code_internal<'a>( +// &'a self, +// onchain_code: RuntimeCode<'a>, +// state: &Backend::State, +// hash: Block::Hash, +// ignore_overrides: bool, +// ) -> sp_blockchain::Result<(RuntimeCode<'a>, RuntimeVersion)> { +// let on_chain_version = self.on_chain_runtime_version(&onchain_code, state)?; +// let code_and_version = if let Some(d) = self.wasm_override.as_ref().as_ref().and_then(|o| { +// if ignore_overrides { +// return None +// } + +// o.get( +// &on_chain_version.spec_version, +// onchain_code.heap_pages, +// &on_chain_version.spec_name, +// ) +// }) { +// tracing::debug!(target: "code-provider::overrides", block = ?hash, "using WASM override"); +// d +// } else if let Some(s) = +// self.wasm_substitutes +// .get(on_chain_version.spec_version, onchain_code.heap_pages, hash) +// { +// tracing::debug!(target: "code-provider::substitutes", block = ?hash, "Using WASM substitute"); +// s +// } else { +// tracing::debug!( +// target: "code-provider", +// block = ?hash, +// "Neither WASM override nor substitute available, using onchain code", +// ); +// (onchain_code, on_chain_version) +// }; + +// Ok(code_and_version) +// } + +// /// Returns the on chain runtime version. +// fn on_chain_runtime_version( +// &self, +// code: &RuntimeCode, +// state: &Backend::State, +// ) -> sp_blockchain::Result { +// let mut overlay = OverlayedChanges::default(); + +// let mut ext = Ext::new(&mut overlay, state, None); + +// self.executor +// .runtime_version(&mut ext, code) +// .map_err(|e| sp_blockchain::Error::VersionInvalid(e.to_string())) +// } +// } diff --git a/internal/client/consensus/common/block_import.go b/internal/client/consensus/common/block_import.go index e5ba1aabdd..285bffab99 100644 --- a/internal/client/consensus/common/block_import.go +++ b/internal/client/consensus/common/block_import.go @@ -123,13 +123,13 @@ type ForkChoiceStrategy interface { type ( // Longest chain fork choice. - LongestChain struct{} + ForkChoiceStrategyLongestChain struct{} // Custom fork choice rule, where true indicates the new block should be the best block. - Custom bool + ForkChoiceStrategyCustom bool ) -func (LongestChain) isForkChoiceStrategy() {} -func (Custom) isForkChoiceStrategy() {} +func (ForkChoiceStrategyLongestChain) isForkChoiceStrategy() {} +func (ForkChoiceStrategyCustom) isForkChoiceStrategy() {} // Data required to import a Block. type BlockImportParams[H runtime.Hash, N runtime.Number, E runtime.Extrinsic, Header runtime.Header[N, H]] struct { @@ -183,3 +183,53 @@ type BlockImportParams[H runtime.Hash, N runtime.Number, E runtime.Extrinsic, He // Cached full header hash (with post-digests applied). PostHash *H } + +// / Get the full header hash (with post-digests applied). +// +// pub fn post_hash(&self) -> Block::Hash { +// if let Some(hash) = self.post_hash { +// hash +// } else { +// self.post_header().hash() +// } +// } +func (b *BlockImportParams[H, N, E, Header]) GetPostHash() H { + if b.PostHash != nil { + return *b.PostHash + } + return b.GetPostHeader().Hash() +} + +/// Get the post header. +// pub fn post_header(&self) -> Block::Header { +// if self.post_digests.is_empty() { +// self.header.clone() +// } else { +// let mut hdr = self.header.clone(); +// for digest_item in &self.post_digests { +// hdr.digest_mut().push(digest_item.clone()); +// } + +// hdr +// } +// } +func (b *BlockImportParams[H, N, E, Header]) GetPostHeader() Header { + if len(b.PostDigests) == 0 { + return b.Header.Clone().(Header) + } + hdr := b.Header.Clone().(Header) + for _, digestItem := range b.PostDigests { + hdr.DigestMut().Push(digestItem) + } + return hdr +} + +// / Check if this block contains state import action +// +// pub fn with_state(&self) -> bool { +// matches!(self.state_action, StateAction::ApplyChanges(StorageChanges::Import(_))) +// } +func (b *BlockImportParams[H, N, E, Header]) WithState() bool { + _, ok := b.StateAction.(StateActionApplyChanges) + return ok +} diff --git a/internal/client/consensus/common/longest_chain.go b/internal/client/consensus/common/longest_chain.go new file mode 100644 index 0000000000..3b03080cb4 --- /dev/null +++ b/internal/client/consensus/common/longest_chain.go @@ -0,0 +1,301 @@ +package common + +import ( + "fmt" + + "github.com/ChainSafe/gossamer/internal/client/api" + chain "github.com/ChainSafe/gossamer/internal/primitives/blockchain" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" +) + +// / Implement Longest Chain Select implementation +// / where 'longest' is defined as the highest number of blocks +// +// pub struct LongestChain { +// backend: Arc, +// _phantom: PhantomData, +// } +type LongestChain[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + backend api.Backend[H, N, Hasher, Header, E] +} + +// impl Clone for LongestChain { +// fn clone(&self) -> Self { +// let backend = self.backend.clone(); +// LongestChain { backend, _phantom: Default::default() } +// } +// } + +// impl LongestChain +// where +// +// B: backend::Backend, +// Block: BlockT, +// +// { +// /// Instantiate a new LongestChain for Backend B +// pub fn new(backend: Arc) -> Self { +// LongestChain { backend, _phantom: Default::default() } +// } +func NewLongestChain[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + backend api.Backend[H, N, Hasher, Header, E], +) LongestChain[H, N, Hasher, Header, E] { + return LongestChain[H, N, Hasher, Header, E]{backend: backend} +} + +// fn best_hash(&self) -> sp_blockchain::Result<::Hash> { +// let info = self.backend.blockchain().info(); +// let import_lock = self.backend.get_import_lock(); +// let best_hash = self +// .backend +// .blockchain() +// .longest_containing(info.best_hash, import_lock)? +// .unwrap_or(info.best_hash); +// Ok(best_hash) +// } +func (lc LongestChain[H, N, Hasher, Header, E]) bestHash() (H, error) { + info := lc.backend.Blockchain().Info() + + var best H + importLock := lc.backend.GetImportLock() + bestHash, err := lc.backend.Blockchain().LongestContaining(info.BestHash, importLock) + if err != nil { + return best, err + } + if bestHash == nil { + return info.BestHash, nil + } + return *bestHash, nil +} + +// fn best_header(&self) -> sp_blockchain::Result<::Header> { +// let best_hash = self.best_hash()?; +// Ok(self +// .backend +// .blockchain() +// .header(best_hash)? +// .expect("given block hash was fetched from block in db; qed")) +// } +func (lc LongestChain[H, N, Hasher, Header, E]) bestHeader() (Header, error) { + var h Header + bestHash, err := lc.bestHash() + if err != nil { + return h, err + } + header, err := lc.backend.Blockchain().Header(bestHash) + if err != nil { + return h, err + } + if header == nil { + panic("given block hash was fetched from block in db; qed") + } + return *header, nil +} + +// / Returns the highest descendant of the given block that is a valid +// / candidate to be finalized. +// / +// / In this context, being a valid target means being an ancestor of +// / the best chain according to the `best_header` method. +// / +// / If `maybe_max_number` is `Some(max_block_number)` the search is +// / limited to block `number <= max_block_number`. In other words +// / as if there were no blocks greater than `max_block_number`. +// +// fn finality_target( +// &self, +// base_hash: Block::Hash, +// maybe_max_number: Option>, +// ) -> sp_blockchain::Result { +func (lc LongestChain[H, N, Hasher, Header, E]) finalityTarget( + baseHash H, + maybeMaxNumber *N, +) (h H, err error) { + // use sp_blockchain::Error::{Application, MissingHeader}; + // let blockchain = self.backend.blockchain(); + blockchain := lc.backend.Blockchain() + + // let mut current_head = self.best_header()?; + // let mut best_hash = current_head.hash(); + currentHead, err := lc.bestHeader() + if err != nil { + return h, err + } + bestHash := currentHead.Hash() + + // let base_header = blockchain + // .header(base_hash)? + // .ok_or_else(|| MissingHeader(base_hash.to_string()))?; + // let base_number = *base_header.number(); + baseHeader, err := blockchain.Header(baseHash) + if err != nil { + return h, err + } + if baseHeader == nil { + return h, fmt.Errorf("%w: %s", chain.ErrMissingHeader, baseHash) + } + baseNumber := (*baseHeader).Number() + + // if let Some(max_number) = maybe_max_number { + if maybeMaxNumber != nil { + // if max_number < base_number { + // let msg = format!( + // "Requested a finality target using max number {} below the base number {}", + // max_number, base_number + // ); + // return Err(Application(msg.into())) + // } + maxNumber := *maybeMaxNumber + if maxNumber < baseNumber { + msg := fmt.Sprintf( + "Requested a finality target using max number %d below the base number %d", + maxNumber, baseNumber, + ) + return h, fmt.Errorf("%w: %s", chain.ErrApplication, msg) + } + + // while current_head.number() > &max_number { + // best_hash = *current_head.parent_hash(); + // current_head = blockchain + // .header(best_hash)? + // .ok_or_else(|| MissingHeader(format!("{best_hash:?}")))?; + // } + for currentHead.Number() > maxNumber { + bestHash = currentHead.ParentHash() + current, err := blockchain.Header(bestHash) + if err != nil { + return h, err + } + if current == nil { + return h, fmt.Errorf("%w: %s", chain.ErrMissingHeader, bestHash) + } + currentHead = *current + } + } + + // while current_head.hash() != base_hash { + // if *current_head.number() < base_number { + // let msg = format!( + // "Requested a finality target using a base {:?} not in the best chain {:?}", + // base_hash, best_hash, + // ); + // return Err(Application(msg.into())) + // } + // let current_hash = *current_head.parent_hash(); + // current_head = blockchain + // .header(current_hash)? + // .ok_or_else(|| MissingHeader(format!("{best_hash:?}")))?; + // } + for currentHead.Hash() != baseHash { + if currentHead.Number() < baseNumber { + msg := fmt.Sprintf( + "Requested a finality target using a base %s not in the best chain %s", + baseHash, bestHash, + ) + return h, fmt.Errorf("%w: %s", chain.ErrApplication, msg) + } + currentHash := currentHead.ParentHash() + current, err := blockchain.Header(currentHash) + if err != nil { + return h, fmt.Errorf("%w: %s", chain.ErrMissingHeader, bestHash) + } + currentHead = *current + } + + // Ok(best_hash) + return bestHash, nil +} + +// fn leaves(&self) -> Result::Hash>, sp_blockchain::Error> { +// self.backend.blockchain().leaves() +// } +func (lc LongestChain[H, N, Hasher, Header, E]) leaves() ([]H, error) { + leaves, err := lc.backend.Blockchain().Leaves() + if err != nil { + return nil, err + } + return leaves, nil +} + +// } + +// #[async_trait::async_trait] +// impl SelectChain for LongestChain +// where +// +// B: backend::Backend, +// Block: BlockT, +// +// { +// async fn leaves(&self) -> Result::Hash>, ConsensusError> { +// LongestChain::leaves(self).map_err(|e| ConsensusError::ChainLookup(e.to_string())) +// } +func (lc LongestChain[H, N, Hasher, Header, E]) Leaves() <-chan common.LeavesError[H] { + leavesChan := make(chan common.LeavesError[H]) + go func() { + defer close(leavesChan) + leaves, err := lc.leaves() + if err != nil { + leavesChan <- common.LeavesError[H]{Error: err} + return + } + leavesChan <- common.LeavesError[H]{Leaves: leaves} + }() + return leavesChan +} + +// async fn best_chain(&self) -> Result<::Header, ConsensusError> { +// LongestChain::best_header(self).map_err(|e| ConsensusError::ChainLookup(e.to_string())) +// } +func (lc LongestChain[H, N, Hasher, Header, E]) BestChain() <-chan common.HeaderError[H, N, Header] { + bestChainChan := make(chan common.HeaderError[H, N, Header]) + go func() { + defer close(bestChainChan) + header, err := lc.bestHeader() + if err != nil { + bestChainChan <- common.HeaderError[H, N, Header]{Error: err} + return + } + bestChainChan <- common.HeaderError[H, N, Header]{Header: header} + }() + return bestChainChan +} + +// async fn finality_target( +// &self, +// base_hash: Block::Hash, +// maybe_max_number: Option>, +// ) -> Result { +// LongestChain::finality_target(self, base_hash, maybe_max_number) +// .map_err(|e| ConsensusError::ChainLookup(e.to_string())) +// } +// } +func (lc LongestChain[H, N, Hasher, Header, E]) FinalityTarget( + baseHash H, + maybeMaxNumber *N, +) <-chan common.HashError[H] { + finalityTargetChan := make(chan common.HashError[H]) + go func() { + defer close(finalityTargetChan) + hash, err := lc.finalityTarget(baseHash, maybeMaxNumber) + if err != nil { + finalityTargetChan <- common.HashError[H]{Error: err} + return + } + finalityTargetChan <- common.HashError[H]{Hash: hash} + }() + return finalityTargetChan +} diff --git a/internal/client/consensus/common/shared-data/shared_data.go b/internal/client/consensus/common/shared-data/shared_data.go new file mode 100644 index 0000000000..78884c985b --- /dev/null +++ b/internal/client/consensus/common/shared-data/shared_data.go @@ -0,0 +1,274 @@ +package shareddata + +import "sync" + +// // / Created by [`SharedDataLocked::release_mutex`]. +// // / +// // / As long as the object isn't dropped, the shared data is locked. It is advised to drop this +// // / object when the shared data doesn't need to be locked anymore. To get access to the shared data +// // / [`Self::upgrade`] is provided. +// // #[must_use = "Shared data will be unlocked on drop!"] +// // +// // pub struct SharedDataLockedUpgradable { +// // shared_data: SharedData, +// // } +// type SharedDataLockedUpgradable[T any] struct { +// sharedData *SharedDataLock[T] +// } + +// // impl SharedDataLockedUpgradable { +// // /// Upgrade to a *real* mutex guard that will give access to the inner data. +// // /// +// // /// Every call to this function will reaquire the mutex again. +// // pub fn upgrade(&mut self) -> MappedMutexGuard { +// // MutexGuard::map(self.shared_data.inner.lock(), |i| &mut i.shared_data) +// // } +// // } +// func (sdlu *SharedDataLockedUpgradable[T]) Upgrade() (*T, func()) { +// sdlu.sharedData.innerMtx.Lock() +// return &sdlu.sharedData.inner.sharedData, func() { +// sdlu.sharedData.innerMtx.Unlock() +// sdlu.Unlock() +// } +// } + +// // impl Drop for SharedDataLockedUpgradable { +// // fn drop(&mut self) { +// // let mut inner = self.shared_data.inner.lock(); +// // // It should not be locked anymore +// // inner.locked = false; + +// // // Notify all waiting threads. +// // self.shared_data.cond_var.notify_all(); +// // } +// // } +// func (sdlu *SharedDataLockedUpgradable[T]) Unlock() { +// sdlu.sharedData.innerMtx.Lock() +// defer sdlu.sharedData.innerMtx.Unlock() +// // It should not be locked anymore +// inner := &sdlu.sharedData.inner +// inner.locked = false + +// // Notify all waiting threads. +// sdlu.sharedData.condVar.Broadcast() +// } + +// // / Created by [`SharedData::shared_data_locked`]. +// // / +// // / As long as this object isn't dropped, the shared data is held in a mutex guard and the shared +// // / data is tagged as locked. Access to the shared data is provided through +// // / [`Deref`](std::ops::Deref) and [`DerefMut`](std::ops::DerefMut). The trick is to use +// // / [`Self::release_mutex`] to release the mutex, but still keep the shared data locked. This means +// // / every other thread trying to access the shared data in this time will need to wait until this +// // / lock is freed. +// // / +// // / If this object is dropped without calling [`Self::release_mutex`], the lock will be dropped +// // / immediately. +// // #[must_use = "Shared data will be unlocked on drop!"] +// // pub struct SharedDataLocked<'a, T> { +// type SharedDataLocked[T any] struct { +// // /// The current active mutex guard holding the inner data. +// // inner: MutexGuard<'a, SharedDataInner>, +// inner *sharedDataInner[T] +// // /// The [`SharedData`] instance that created this instance. +// // /// +// // /// This instance is only taken on drop or when calling [`Self::release_mutex`]. +// // shared_data: Option>, +// sharedData *SharedDataLock[T] +// } + +// func (sdl *SharedDataLocked[T]) ReleaseMutex() SharedDataLockedUpgradable[T] { +// /// Release the mutex, but keep the shared data locked. +// sharedData := sdl.sharedData +// if sharedData == nil { +// panic("shared data is already released") +// } +// sdl.sharedData = nil +// return SharedDataLockedUpgradable[T]{sharedData: sharedData} +// } + +// // impl<'a, T> Drop for SharedDataLocked<'a, T> { +// // fn drop(&mut self) { +// // if let Some(shared_data) = self.shared_data.take() { +// // // If the `shared_data` is still set, it means [`Self::release_mutex`] wasn't +// // // called and the lock should be released. +// // self.inner.locked = false; + +// // // Notify all waiting threads about the released lock. +// // shared_data.cond_var.notify_all(); +// // } +// // } +// // } +// func (sdl *SharedDataLocked[T]) Unlock() { +// if sdl.sharedData != nil { +// sharedData := sdl.sharedData +// sdl.sharedData = nil +// // If the `shared_data` is still set, it means [`Self::release_mutex`] wasn't +// // called and the lock should be released. +// sdl.inner.locked = false + +// // Notify all waiting threads about the released lock. +// sharedData.condVar.Broadcast() +// } +// } + +// // / Holds the shared data and if the shared data is currently locked. +// // / +// // / For more information see [`SharedData`]. +// // struct SharedDataInner { +// type sharedDataInner[T any] struct { +// // /// The actual shared data that is protected here against concurrent access. +// // shared_data: T, +// sharedData T +// // /// Is `shared_data` currently locked and can not be accessed? +// // locked: bool, +// locked bool +// } + +// type SharedDataLock[T any] struct { +// inner sharedDataInner[T] +// innerMtx sync.Mutex +// condVar *sync.Cond +// } + +// // / Create a new instance of [`SharedData`] to share the given `shared_data`. +// // +// // pub fn new(shared_data: T) -> Self { +// // Self { +// // inner: Arc::new(Mutex::new(SharedDataInner { shared_data, locked: false })), +// // cond_var: Default::default(), +// // } +// // } +// func NewSharedDataLock[T any](sharedData T) *SharedDataLock[T] { +// sd := SharedDataLock[T]{ +// inner: sharedDataInner[T]{ +// sharedData: sharedData, +// locked: false, +// }, +// } +// sd.condVar = sync.NewCond(&sd.innerMtx) +// return &sd +// } + +// // / Acquire access to the shared data. +// // / +// // / This will give mutable access to the shared data. After the returned mutex guard is dropped, +// // / the shared data is accessible by other threads. So, this function should be used when +// // / reading/writing of the shared data in a local context is required. +// // / +// // / When requiring to lock shared data for some longer time, even with temporarily releasing the +// // / lock, [`Self::shared_data_locked`] should be used. +// // pub fn shared_data(&self) -> MappedMutexGuard { +// // let mut guard = self.inner.lock(); +// // while guard.locked { +// // self.cond_var.wait(&mut guard); +// // } +// // +// // debug_assert!(!guard.locked); +// // +// // MutexGuard::map(guard, |i| &mut i.shared_data) +// // } + +// // let mut guard = self.inner.lock(); + +// // while guard.locked { +// // self.cond_var.wait(&mut guard); +// // } + +// // debug_assert!(!guard.locked); + +// // MutexGuard::map(guard, |i| &mut i.shared_data) +// // } +// func (sd *SharedDataLock[T]) SharedData() T { +// sd.innerMtx.Lock() +// defer sd.innerMtx.Unlock() + +// for sd.inner.locked { +// sd.condVar.Wait() +// } +// if sd.inner.locked { +// panic("shared data is already locked") +// } + +// return sd.inner.sharedData +// } + +// // / Acquire access to the shared data and lock it. +// // / +// // / This will give mutable access to the shared data. The returned [`SharedDataLocked`] +// // / provides the function [`SharedDataLocked::release_mutex`] to release the mutex, but +// // / keeping the data locked. This is useful in async contexts for example where the data needs +// // / to be locked, but a mutex guard can not be held. +// // / +// // / For an example see [`SharedData`]. +// // pub fn shared_data_locked(&self) -> SharedDataLocked { +// func (sd *SharedDataLock[T]) SharedDataLocked() SharedDataLocked[T] { +// // let mut guard = self.inner.lock(); +// sd.innerMtx.Lock() +// defer sd.innerMtx.Unlock() + +// // while guard.locked { +// // self.cond_var.wait(&mut guard); +// // } +// for sd.inner.locked { +// sd.condVar.Wait() +// } + +// // debug_assert!(!guard.locked); +// if sd.inner.locked { +// panic("shared data is already locked") +// } +// // guard.locked = true; +// sd.inner.locked = true + +// // SharedDataLocked { inner: guard, shared_data: Some(self.clone()) } +// return SharedDataLocked[T]{ +// inner: &sd.inner, +// sharedData: sd, +// } +// } + +type SharedData[T any] struct { + inner T + innerMtx sync.RWMutex +} + +func NewSharedData[T any](sharedData T) *SharedData[T] { + sd := SharedData[T]{ + inner: sharedData, + } + return &sd +} + +func (sd *SharedData[T]) Data() T { + sd.innerMtx.RLock() + defer sd.innerMtx.RUnlock() + + return sd.inner +} + +func (sd *SharedData[T]) DataMut() (*T, func()) { + sd.innerMtx.Lock() + return &sd.inner, sd.innerMtx.Unlock +} + +func (sd *SharedData[T]) Locked() SharedDataLocked[T] { + sd.innerMtx.Lock() + return SharedDataLocked[T]{sharedData: sd} +} + +type SharedDataLocked[T any] struct { + sharedData *SharedData[T] +} + +func (sdl SharedDataLocked[T]) Data() T { + return sdl.sharedData.inner +} + +func (sdl SharedDataLocked[T]) MutRef() *T { + return &sdl.sharedData.inner +} + +func (sdl SharedDataLocked[T]) Unlock() { + sdl.sharedData.innerMtx.Unlock() +} diff --git a/internal/client/consensus/common/shared-data/shared_data_test.go b/internal/client/consensus/common/shared-data/shared_data_test.go new file mode 100644 index 0000000000..6b099e88c0 --- /dev/null +++ b/internal/client/consensus/common/shared-data/shared_data_test.go @@ -0,0 +1,138 @@ +package shareddata + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSharedData(t *testing.T) { + // // fn shared_data_locking_works() { + // t.Run("shared_data_locking_works", func(t *testing.T) { + // // const THREADS: u32 = 100; + // // let shared_data = SharedData::new(0u32); + // const numThreads = 100 + // sharedData := NewSharedDataLock[uint32](0) + + // // let lock = shared_data.shared_data_locked(); + // lock := sharedData.SharedDataLocked() + + // // for i in 0..THREADS { + // for i := 0; i < numThreads; i++ { + // // let data = shared_data.clone(); + // data := sharedData + // // std::thread::spawn(move || { + // go func(i int) { + // // if i % 2 == 1 { + // // *data.shared_data() += 1; + // // } else { + // // let mut lock = data.shared_data_locked().release_mutex(); + // // // Give the other threads some time to wake up + // // std::thread::sleep(std::time::Duration::from_millis(10)); + // // *lock.upgrade() += 1; + // // } + // // }); + // lock := data.SharedDataLocked() + // defer lock.Unlock() + // released := lock.ReleaseMutex() + // time.Sleep(10 * time.Millisecond) + // ref, unlock := released.Upgrade() + // defer unlock() + // *ref += 1 + // // unlock() + // // released.Unlock() + // // lock.Unlock() + // }(i) + // } + + // // let lock = lock.release_mutex(); + // // std::thread::sleep(std::time::Duration::from_millis(100)); + // // drop(lock); + // released := lock.ReleaseMutex() + // // ensure that this is still 0 + // time.Sleep(100 * time.Millisecond) + // require.Equal(t, uint32(0), released.sharedData.inner.sharedData) + + // released.Unlock() + // lock.Unlock() + + // // while *shared_data.shared_data() < THREADS { + // for { + // // std::thread::sleep(std::time::Duration::from_millis(100)); + // sd := sharedData.SharedData() + // t.Logf("sd: %d", sd) + // if sd >= numThreads { + // break + // } + // time.Sleep(1 * time.Millisecond) + // } + + // }) + + t.Run("shared_data_locking_works", func(t *testing.T) { + const numThreads = 100 + sharedData := NewSharedData[uint32](0) + + ref, unlock := sharedData.DataMut() + + for i := 0; i < numThreads; i++ { + go func(i int) { + ref, unlock := sharedData.DataMut() + defer unlock() + // Give the other threads some time to wake up + time.Sleep(10 * time.Millisecond) + *ref += 1 + }(i) + } + + time.Sleep(100 * time.Millisecond) + // ensure that this is still 0 + require.Equal(t, uint32(0), *ref) + unlock() + + for { + sd := sharedData.Data() + t.Logf("sd: %d", sd) + if sd >= numThreads { + break + } + time.Sleep(100 * time.Millisecond) + } + + }) + + t.Run("shared_data_locking_works_with_lock", func(t *testing.T) { + const numThreads = 100 + sharedData := NewSharedData[uint32](0) + + locked := sharedData.Locked() + ref := locked.MutRef() + + for i := 0; i < numThreads; i++ { + go func(i int) { + locked := sharedData.Locked() + defer locked.Unlock() + ref := locked.MutRef() + // Give the other threads some time to wake up + time.Sleep(10 * time.Millisecond) + *ref += 1 + }(i) + } + + time.Sleep(100 * time.Millisecond) + // ensure that this is still 0 + require.Equal(t, uint32(0), *ref) + locked.Unlock() + + for { + sd := sharedData.Data() + t.Logf("sd: %d", sd) + if sd >= numThreads { + break + } + time.Sleep(100 * time.Millisecond) + } + + }) +} diff --git a/internal/client/consensus/grandpa/authorities.go b/internal/client/consensus/grandpa/authorities.go index 81aa39f688..d8244fe60b 100644 --- a/internal/client/consensus/grandpa/authorities.go +++ b/internal/client/consensus/grandpa/authorities.go @@ -6,12 +6,14 @@ package grandpa import ( "errors" "fmt" - "sync" + "slices" + + shareddata "github.com/ChainSafe/gossamer/internal/client/consensus/common/shared-data" pgrandpa "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" + forktree "github.com/ChainSafe/gossamer/internal/utils/fork-tree" grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "golang.org/x/exp/constraints" - "golang.org/x/exp/slices" ) var ( @@ -58,22 +60,31 @@ type status[H comparable, N constraints.Unsigned] struct { // SharedAuthoritySet A shared authority set type SharedAuthoritySet[H comparable, N constraints.Unsigned] struct { - mtx sync.Mutex - inner AuthoritySet[H, N] + // mtx sync.Mutex + // inner AuthoritySet[H, N] + inner shareddata.SharedData[AuthoritySet[H, N]] +} + +// / Get the current set ID. This is incremented every time the set changes. +func (sas *SharedAuthoritySet[H, N]) SetID() uint64 { + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.SetID + authSet := sas.inner.Data() + return authSet.SetID } // CurrentAuthorities will get the current authorities and their weights (for the current set ID). -func (sas *SharedAuthoritySet[H, N]) CurrentAuthorities() grandpa.VoterSet[string] { - sas.mtx.Lock() - defer sas.mtx.Unlock() - idWeights := make([]grandpa.IDWeight[string], len(sas.inner.CurrentAuthorities)) - for i, auth := range sas.inner.CurrentAuthorities { - idWeights[i] = grandpa.IDWeight[string]{ - ID: string(auth.AuthorityID), +func (sas *SharedAuthoritySet[H, N]) CurrentAuthorities() grandpa.VoterSet[pgrandpa.AuthorityID] { + authSet := sas.inner.Data() + idWeights := make([]grandpa.IDWeight[pgrandpa.AuthorityID], len(authSet.CurrentAuthorities)) + for i, auth := range authSet.CurrentAuthorities { + idWeights[i] = grandpa.IDWeight[pgrandpa.AuthorityID]{ + ID: auth.AuthorityID, Weight: uint64(auth.AuthorityWeight), } } - voterSet := grandpa.NewVoterSet[string](idWeights) + voterSet := grandpa.NewVoterSet[pgrandpa.AuthorityID](idWeights) if voterSet == nil { panic("CurrentAuthorities is non-empty and weights are non-zero; constructor and all" + " mutating operations on AuthoritySet ensure this.") @@ -83,51 +94,65 @@ func (sas *SharedAuthoritySet[H, N]) CurrentAuthorities() grandpa.VoterSet[strin // Current Get the current set id and a reference to the current authority set. func (sas *SharedAuthoritySet[H, N]) Current() (uint64, pgrandpa.AuthorityList) { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.current() + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.current() + authSet := sas.inner.Data() + return authSet.current() } func (sas *SharedAuthoritySet[H, N]) revert() { //nolint //skipcq: SCC-U1000 - sas.mtx.Lock() - defer sas.mtx.Unlock() - sas.inner.revert() + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // sas.inner.revert() + authSet := sas.inner.Data() + authSet.revert() } func (sas *SharedAuthoritySet[H, N]) nextChange(bestHash H, //nolint //skipcq: SCC-U1000 isDescendentOf IsDescendentOf[H]) (*HashNumber[H, N], error) { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.nextChange(bestHash, isDescendentOf) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.nextChange(bestHash, isDescendentOf) + authSet := sas.inner.Data() + return authSet.nextChange(bestHash, isDescendentOf) } func (sas *SharedAuthoritySet[H, N]) addStandardChange(pending PendingChange[H, N], //nolint //skipcq: SCC-U1000 isDescendentOf IsDescendentOf[H]) error { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.addStandardChange(pending, isDescendentOf) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.addStandardChange(pending, isDescendentOf) + authSet := sas.inner.Data() + return authSet.addStandardChange(pending, isDescendentOf) } func (sas *SharedAuthoritySet[H, N]) addForcedChange(pending PendingChange[H, N], //nolint //skipcq: SCC-U1000 isDescendentOf IsDescendentOf[H]) error { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.addForcedChange(pending, isDescendentOf) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.addForcedChange(pending, isDescendentOf) + authSet := sas.inner.Data() + return authSet.addForcedChange(pending, isDescendentOf) } func (sas *SharedAuthoritySet[H, N]) addPendingChange(pending PendingChange[H, N], //nolint //skipcq: SCC-U1000 isDescendentOf IsDescendentOf[H]) error { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.addPendingChange(pending, isDescendentOf) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.addPendingChange(pending, isDescendentOf) + authSet := sas.inner.Data() + return authSet.addPendingChange(pending, isDescendentOf) } // PendingChanges inspects pending changes. Standard pending changes are iterated first, and the changes in the roots // are traversed in pre-order, afterwards all forced changes are iterated. func (sas *SharedAuthoritySet[H, N]) PendingChanges() []PendingChange[H, N] { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.pendingChanges() + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.pendingChanges() + authSet := sas.inner.Data() + return authSet.pendingChanges() } // currentLimit will get the earliest limit-block number, if any. If there are pending changes across different forks, @@ -137,9 +162,11 @@ func (sas *SharedAuthoritySet[H, N]) PendingChanges() []PendingChange[H, N] { // Only standard changes are taken into account for the current limit, since any existing forced change should preclude // the voter from voting. func (sas *SharedAuthoritySet[H, N]) currentLimit(min N) (limit *N) { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.currentLimit(min) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.currentLimit(min) + authSet := sas.inner.Data() + return authSet.currentLimit(min) } func (sas *SharedAuthoritySet[H, N]) applyForcedChanges( //nolint:unused @@ -148,9 +175,11 @@ func (sas *SharedAuthoritySet[H, N]) applyForcedChanges( //nolint:unused isDescendentOf IsDescendentOf[H], // TODO: telemtry, ) (newSet *appliedChanges[H, N], err error) { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.applyForcedChanges(bestHash, bestNumber, isDescendentOf) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.applyForcedChanges(bestHash, bestNumber, isDescendentOf) + authSet := sas.inner.Data() + return authSet.applyForcedChanges(bestHash, bestNumber, isDescendentOf) } // applyStandardChanges will apply or prune any pending transitions based on a finality trigger. This method ensures @@ -167,9 +196,11 @@ func (sas *SharedAuthoritySet[H, N]) applyStandardChanges( initialSync bool, // TODO: telemetry, ) (status[H, N], error) { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.applyStandardChanges(finalisedHash, finalisedNumber, isDescendentOf, initialSync) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.applyStandardChanges(finalisedHash, finalisedNumber, isDescendentOf, initialSync) + authSet := sas.inner.Data() + return authSet.applyStandardChanges(finalisedHash, finalisedNumber, isDescendentOf, initialSync) } // EnactsStandardChange Check whether the given finalised block number enacts any standard authority set change @@ -181,9 +212,11 @@ func (sas *SharedAuthoritySet[H, N]) applyStandardChanges( func (sas *SharedAuthoritySet[H, N]) EnactsStandardChange(finalisedHash H, finalisedNumber N, isDescendentOf IsDescendentOf[H]) (*bool, error) { - sas.mtx.Lock() - defer sas.mtx.Unlock() - return sas.inner.EnactsStandardChange(finalisedHash, finalisedNumber, isDescendentOf) + // sas.mtx.Lock() + // defer sas.mtx.Unlock() + // return sas.inner.EnactsStandardChange(finalisedHash, finalisedNumber, isDescendentOf) + authSet := sas.inner.Data() + return authSet.EnactsStandardChange(finalisedHash, finalisedNumber, isDescendentOf) } // AuthoritySet A set of authorities. @@ -195,7 +228,7 @@ type AuthoritySet[H comparable, N constraints.Unsigned] struct { // Tree of pending standard changes across forks. Standard changes are // enacted on finality and must be enacted (i.e. finalised) in-order across // a given branch - PendingStandardChanges ChangeTree[H, N] + PendingStandardChanges forktree.ForkTree[H, N, PendingChange[H, N]] // Pending forced changes across different forks (at most one per fork). // Forced changes are enacted on block depth (not finality), for this // reason only one forced HashNumber should exist per fork. When trying to @@ -242,7 +275,7 @@ func NewGenesisAuthoritySet[H comparable, N constraints.Unsigned]( func NewAuthoritySet[H comparable, N constraints.Unsigned]( authorities pgrandpa.AuthorityList, setID uint64, - pendingStandardChanges ChangeTree[H, N], + pendingStandardChanges forktree.ForkTree[H, N, PendingChange[H, N]], pendingForcedChanges []PendingChange[H, N], authoritySetChanges AuthoritySetChanges[N], ) (authSet *AuthoritySet[H, N], err error) { @@ -259,6 +292,10 @@ func NewAuthoritySet[H comparable, N constraints.Unsigned]( }, nil } +func (authSet *AuthoritySet[H, N]) Clone() AuthoritySet[H, N] { + panic("unimpl") +} + // current retrieves the current set id and a reference to the current authority set. func (authSet *AuthoritySet[H, N]) current() (uint64, pgrandpa.AuthorityList) { return authSet.SetID, authSet.CurrentAuthorities @@ -295,7 +332,7 @@ func (authSet *AuthoritySet[H, N]) nextChange(bestHash H, //skipcq: RVV-B0001 var standard *HashNumber[H, N] for _, changeNode := range authSet.PendingStandardChanges.Roots() { - c := changeNode.Change + c := changeNode.Data isDesc, err := isDescendentOf(c.CanonHash, bestHash) if err != nil { return nil, err @@ -344,7 +381,8 @@ func (authSet *AuthoritySet[H, N]) addStandardChange( logger.Debugf( "There are now %d alternatives for the next pending standard HashNumber (roots), "+ "and a total of %d pending standard changes (across all forks)", - len(authSet.PendingStandardChanges.Roots()), len(authSet.PendingStandardChanges.PendingChanges()), + len(authSet.PendingStandardChanges.Roots()), + len(slices.Collect(authSet.PendingStandardChanges.Iter())), ) return nil @@ -437,10 +475,10 @@ func (authSet *AuthoritySet[H, N]) addPendingChange( return errInvalidAuthoritySet } - switch pending.DelayKind.Value.(type) { - case Finalized: + switch pending.DelayKind.(type) { + case delayKindFinalized: return authSet.addStandardChange(pending, isDescendentOf) - case Best[N]: + case delayKindBest[N]: return authSet.addForcedChange(pending, isDescendentOf) default: panic("DelayKind is invalid type") @@ -451,7 +489,10 @@ func (authSet *AuthoritySet[H, N]) addPendingChange( // roots are traversed in pre-order, afterwards all forced changes are iterated. func (authSet *AuthoritySet[H, N]) pendingChanges() []PendingChange[H, N] { //skipcq: RVV-B0001 // get everything from standard HashNumber roots - changes := authSet.PendingStandardChanges.PendingChanges() + var changes []PendingChange[H, N] + for change := range authSet.PendingStandardChanges.Iter() { + changes = append(changes, change.Data) + } // append forced changes changes = append(changes, authSet.PendingForcedChanges...) @@ -468,7 +509,7 @@ func (authSet *AuthoritySet[H, N]) pendingChanges() []PendingChange[H, N] { //sk func (authSet *AuthoritySet[H, N]) currentLimit(min N) (limit *N) { roots := authSet.PendingStandardChanges.Roots() for i := 0; i < len(roots); i++ { - effectiveNumber := roots[i].Change.EffectiveNumber() + effectiveNumber := roots[i].Data.EffectiveNumber() if effectiveNumber >= min { if limit == nil { limit = &effectiveNumber @@ -510,12 +551,12 @@ func (authSet *AuthoritySet[H, N]) applyForcedChanges(bestHash H, //skipcq: RVV return nil, err } if change.CanonHash == bestHash || isDesc { - switch delayKindType := change.DelayKind.Value.(type) { - case Best[N]: - medianLastFinalized := delayKindType.medianLastFinalized + switch delayKindType := change.DelayKind.(type) { + case delayKindBest[N]: + medianLastFinalized := delayKindType.MedianLastFinalized roots := authSet.PendingStandardChanges.Roots() for _, standardChangeNode := range roots { - standardChange := standardChangeNode.Change + standardChange := standardChangeNode.Data isDescStandard, err := isDescendentOf(standardChange.CanonHash, change.CanonHash) if err != nil { @@ -542,7 +583,7 @@ func (authSet *AuthoritySet[H, N]) applyForcedChanges(bestHash H, //skipcq: RVV AuthoritySet[H, N]{ CurrentAuthorities: change.NextAuthorities, SetID: authSet.SetID + 1, - PendingStandardChanges: NewChangeTree[H, N](), // new set, new changes + PendingStandardChanges: forktree.NewForkTree[H, N, PendingChange[H, N]](), // new set, new changes PendingForcedChanges: []PendingChange[H, N]{}, AuthoritySetChanges: authSetChanges, }, @@ -573,24 +614,21 @@ func (authSet *AuthoritySet[H, N]) applyStandardChanges( //skipcq: RVV-B0001 // TODO telemetry here is just a place holder, replace with real status := status[H, N]{} - finalisationResult, err := authSet.PendingStandardChanges.FinalizeWithDescendentIf(&finalisedHash, + finalisationResult, err := authSet.PendingStandardChanges.FinalizeWithDescendentIf(finalisedHash, finalisedNumber, isDescendentOf, - func(change *PendingChange[H, N]) bool { + func(change PendingChange[H, N]) bool { return change.EffectiveNumber() <= finalisedNumber }) if err != nil { return status, err } - finalisationResultVal, err := finalisationResult.Value() - if err != nil { - return status, err - } - switch val := finalisationResultVal.(type) { - case unchanged: + switch val := finalisationResult.(type) { + case forktree.FinalizationResultUnchanged: return status, nil - case changed[H, N]: + case forktree.FinalizationResultChanged[PendingChange[H, N]]: + change := val // Changed Case status.Changed = true @@ -610,18 +648,18 @@ func (authSet *AuthoritySet[H, N]) applyStandardChanges( //skipcq: RVV-B0001 } } - if val.value != nil { + if change.Value != nil { var level func(format string, args ...interface{}) = logger.Debugf if initialSync { level = logger.Infof } - level("👴 Applying authority set scheduled at block #%d", val.value.CanonHeight) + level("👴 Applying authority set scheduled at block #%d", change.Value.CanonHeight) // TODO add telemetry // Store the setID together with the last block_number for the set authSet.AuthoritySetChanges.append(authSet.SetID, finalisedNumber) - authSet.CurrentAuthorities = val.value.NextAuthorities + authSet.CurrentAuthorities = change.Value.NextAuthorities authSet.SetID++ status.NewSetBlock = &HashNumber[H, N]{ @@ -645,10 +683,10 @@ func (authSet *AuthoritySet[H, N]) applyStandardChanges( //skipcq: RVV-B0001 // first hash (base). func (authSet *AuthoritySet[H, N]) EnactsStandardChange( //skipcq: RVV-B0001 finalisedHash H, finalisedNumber N, isDescendentOf IsDescendentOf[H]) (*bool, error) { - applied, err := authSet.PendingStandardChanges.FinalizesAnyWithDescendentIf(&finalisedHash, + applied, err := authSet.PendingStandardChanges.FinalizesAnyWithDescendentIf(finalisedHash, finalisedNumber, isDescendentOf, - func(change *PendingChange[H, N]) bool { + func(change PendingChange[H, N]) bool { return change.EffectiveNumber() == finalisedNumber }) if err != nil { @@ -657,31 +695,38 @@ func (authSet *AuthoritySet[H, N]) EnactsStandardChange( //skipcq: RVV-B0001 return applied, nil } -// delayKinds Kinds of delays for pending changes. -type delayKinds[N constraints.Unsigned] interface { - Finalized | Best[N] +// delayKind are the inds of delays for pending changes. +type delayKind interface { + isDelayKind() +} +type delayKindTypes[N constraints.Unsigned] interface { + delayKindFinalized | delayKindBest[N] } // delayKind struct to represent delayedKinds -type delayKind struct { - Value interface{} +type delayKindVDT struct { + inner interface{} } -func newDelayKind[N constraints.Unsigned, T delayKinds[N]](val T) delayKind { - return delayKind{ - Value: val, +func newDelayKind[N constraints.Unsigned, T delayKindTypes[N]](val T) delayKindVDT { + return delayKindVDT{ + inner: val, } } // Finalized Depth in finalised chain. -type Finalized struct{} +type delayKindFinalized struct{} + +func (delayKindFinalized) isDelayKind() {} // Best depth in best chain. The median last finalised block is calculated at the time the // HashNumber was signalled. -type Best[N constraints.Unsigned] struct { - medianLastFinalized N +type delayKindBest[N constraints.Unsigned] struct { + MedianLastFinalized N } +func (delayKindBest[N]) isDelayKind() {} + // PendingChange is a pending HashNumber to the authority set. // // This will be applied when the announcing block is at some depth within the finalised or unfinalised chain. diff --git a/internal/client/consensus/grandpa/authorities_test.go b/internal/client/consensus/grandpa/authorities_test.go index efcf68a4ff..b54cd142e8 100644 --- a/internal/client/consensus/grandpa/authorities_test.go +++ b/internal/client/consensus/grandpa/authorities_test.go @@ -8,6 +8,7 @@ import ( pgrandpa "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa/app" + forktree "github.com/ChainSafe/gossamer/internal/utils/fork-tree" "github.com/stretchr/testify/require" ) @@ -29,17 +30,17 @@ func isDescendentof[H comparable](f IsDescendentOf[H]) IsDescendentOf[H] { } func TestDelayKind(t *testing.T) { - finalizedKind := Finalized{} + finalizedKind := delayKindFinalized{} delayKind := newDelayKind[uint](finalizedKind) - _, isFinalizedType := delayKind.Value.(Finalized) - require.True(t, isFinalizedType) + _, isdelayKindFinalizedType := delayKind.inner.(delayKindFinalized) + require.True(t, isdelayKindFinalizedType) - medLastFinalized := uint(3) - bestKind := Best[uint]{medianLastFinalized: medLastFinalized} + medLastdelayKindFinalized := uint(3) + bestKind := delayKindBest[uint]{MedianLastFinalized: medLastdelayKindFinalized} delayKind = newDelayKind[uint](bestKind) - best, isBestType := delayKind.Value.(Best[uint]) - require.True(t, isBestType) - require.Equal(t, medLastFinalized, best.medianLastFinalized) + best, isdelayKindBestType := delayKind.inner.(delayKindBest[uint]) + require.True(t, isdelayKindBestType) + require.Equal(t, medLastdelayKindFinalized, best.MedianLastFinalized) } func newTestPublic(t *testing.T, index uint8) app.Public { @@ -62,15 +63,14 @@ func TestCurrentLimitFiltersMin(t *testing.T) { AuthorityWeight: 1, }, } - finalisedKind := Finalized{} - delayKind := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} pendingChange1 := PendingChange[string, uint]{ NextAuthorities: currentAuthorities, Delay: 0, CanonHeight: 1, CanonHash: "a", - DelayKind: delayKind, + DelayKind: finalisedKind, } pendingChange2 := PendingChange[string, uint]{ @@ -78,13 +78,13 @@ func TestCurrentLimitFiltersMin(t *testing.T) { Delay: 0, CanonHeight: 2, CanonHash: "b", - DelayKind: delayKind, + DelayKind: finalisedKind, } authorities := AuthoritySet[string, uint]{ CurrentAuthorities: currentAuthorities, SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -109,16 +109,13 @@ func TestChangesIteratedInPreOrder(t *testing.T) { }, } - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) - - bestKind := Best[uint]{} - delayKindBest := newDelayKind[uint](bestKind) + finalisedKind := delayKindFinalized{} + bestKind := delayKindBest[uint]{} authorities := AuthoritySet[string, uint]{ CurrentAuthorities: currentAuthorities, SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -128,7 +125,7 @@ func TestChangesIteratedInPreOrder(t *testing.T) { Delay: 10, CanonHeight: 5, CanonHash: "hash_a", - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeB := PendingChange[string, uint]{ @@ -136,7 +133,7 @@ func TestChangesIteratedInPreOrder(t *testing.T) { Delay: 0, CanonHeight: 5, CanonHash: "hash_b", - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeC := PendingChange[string, uint]{ @@ -144,7 +141,7 @@ func TestChangesIteratedInPreOrder(t *testing.T) { Delay: 5, CanonHeight: 10, CanonHash: "hash_c", - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } err := authorities.addPendingChange(changeA, staticIsDescendentOf[string](false)) @@ -169,7 +166,7 @@ func TestChangesIteratedInPreOrder(t *testing.T) { Delay: 2, CanonHeight: 1, CanonHash: hashD, - DelayKind: delayKindBest, + DelayKind: bestKind, } changeE := PendingChange[string, uint]{ @@ -177,7 +174,7 @@ func TestChangesIteratedInPreOrder(t *testing.T) { Delay: 2, CanonHeight: 0, CanonHash: "hash_e", - DelayKind: delayKindBest, + DelayKind: bestKind, } err = authorities.addPendingChange(changeD, staticIsDescendentOf[string](false)) @@ -196,7 +193,7 @@ func TestChangesIteratedInPreOrder(t *testing.T) { func TestApplyChange(t *testing.T) { authorities := AuthoritySet[string, uint]{ SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -215,15 +212,14 @@ func TestApplyChange(t *testing.T) { }, } - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} changeA := PendingChange[string, uint]{ NextAuthorities: setA, Delay: 10, CanonHeight: 5, CanonHash: hashA, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeB := PendingChange[string, uint]{ @@ -231,7 +227,7 @@ func TestApplyChange(t *testing.T) { Delay: 10, CanonHeight: 5, CanonHash: hashB, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } err := authorities.addPendingChange(changeA, staticIsDescendentOf[string](true)) @@ -260,7 +256,7 @@ func TestApplyChange(t *testing.T) { panic("unreachable") } }), - true, + false, ) require.NoError(t, err) @@ -307,10 +303,10 @@ func TestApplyChange(t *testing.T) { require.Equal(t, authorities.AuthoritySetChanges, AuthoritySetChanges[uint]{expChange}) } -func TestDisallowMultipleChangesBeingFinalizedAtOnce(t *testing.T) { +func TestDisallowMultipleChangesBeingdelayKindFinalizedAtOnce(t *testing.T) { authorities := AuthoritySet[string, uint]{ SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -329,15 +325,14 @@ func TestDisallowMultipleChangesBeingFinalizedAtOnce(t *testing.T) { }, } - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} changeA := PendingChange[string, uint]{ NextAuthorities: setA, Delay: 10, CanonHeight: 5, CanonHash: hashA, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeC := PendingChange[string, uint]{ @@ -345,7 +340,7 @@ func TestDisallowMultipleChangesBeingFinalizedAtOnce(t *testing.T) { Delay: 10, CanonHeight: 30, CanonHash: hashC, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } err := authorities.addPendingChange(changeA, staticIsDescendentOf[string](true)) @@ -376,7 +371,7 @@ func TestDisallowMultipleChangesBeingFinalizedAtOnce(t *testing.T) { true, ) - require.ErrorIs(t, err, errUnfinalisedAncestor) + require.ErrorIs(t, err, forktree.ErrUnfinalizedAncestor) require.Equal(t, AuthoritySetChanges[uint]{}, authorities.AuthoritySetChanges) status, err := authorities.applyStandardChanges( @@ -433,7 +428,7 @@ func TestDisallowMultipleChangesBeingFinalizedAtOnce(t *testing.T) { func TestEnactsStandardChangeWorks(t *testing.T) { authorities := AuthoritySet[string, uint]{ SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -445,15 +440,14 @@ func TestEnactsStandardChangeWorks(t *testing.T) { }, } - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} changeA := PendingChange[string, uint]{ NextAuthorities: setA, Delay: 10, CanonHeight: 5, CanonHash: hashA, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeB := PendingChange[string, uint]{ @@ -461,7 +455,7 @@ func TestEnactsStandardChangeWorks(t *testing.T) { Delay: 10, CanonHeight: 20, CanonHash: hashB, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } err := authorities.addPendingChange(changeA, staticIsDescendentOf[string](false)) @@ -509,7 +503,7 @@ func TestEnactsStandardChangeWorks(t *testing.T) { func TestForceChanges(t *testing.T) { authorities := AuthoritySet[string, uint]{ SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -528,18 +522,16 @@ func TestForceChanges(t *testing.T) { }, } - finalisedKindA := Best[uint]{42} - delayKindFinalizedA := newDelayKind[uint](finalisedKindA) + finalisedKindA := delayKindBest[uint]{42} - finalisedKindB := Best[uint]{0} - delayKindFinalizedB := newDelayKind[uint](finalisedKindB) + finalisedKindB := delayKindBest[uint]{0} changeA := PendingChange[string, uint]{ NextAuthorities: setA, Delay: 10, CanonHeight: 5, CanonHash: hashA, - DelayKind: delayKindFinalizedA, + DelayKind: finalisedKindA, } changeB := PendingChange[string, uint]{ @@ -547,7 +539,7 @@ func TestForceChanges(t *testing.T) { Delay: 10, CanonHeight: 5, CanonHash: hashB, - DelayKind: delayKindFinalizedB, + DelayKind: finalisedKindB, } err := authorities.addPendingChange(changeA, staticIsDescendentOf[string](false)) @@ -569,7 +561,7 @@ func TestForceChanges(t *testing.T) { Delay: 3, CanonHeight: 8, CanonHash: "hash_a8", - DelayKind: delayKindFinalizedB, + DelayKind: finalisedKindB, } isDescOfA := isDescendentof(func(h1 string, _ string) (bool, error) { @@ -600,7 +592,7 @@ func TestForceChanges(t *testing.T) { set: AuthoritySet[string, uint]{ CurrentAuthorities: setA, SetID: 1, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{ setIDNumber[uint]{ @@ -620,7 +612,7 @@ func TestForceChangesWithNoDelay(t *testing.T) { // NOTE: this is a regression test authorities := AuthoritySet[string, uint]{ SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -632,8 +624,7 @@ func TestForceChangesWithNoDelay(t *testing.T) { }, } - finalisedKind := Best[uint]{0} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindBest[uint]{0} // we create a forced HashNumber with no Delay changeA := PendingChange[string, uint]{ @@ -641,7 +632,7 @@ func TestForceChangesWithNoDelay(t *testing.T) { Delay: 0, CanonHeight: 5, CanonHash: hashA, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } // and import it @@ -661,7 +652,7 @@ func TestForceChangesWithNoDelay(t *testing.T) { func TestForceChangesBlockedByStandardChanges(t *testing.T) { authorities := AuthoritySet[string, uint]{ SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -673,8 +664,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { }, } - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} // effective at #15 changeA := PendingChange[string, uint]{ @@ -682,7 +672,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { Delay: 5, CanonHeight: 10, CanonHash: hashA, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } // effective #20 @@ -691,7 +681,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { Delay: 0, CanonHeight: 20, CanonHash: hashB, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } // effective at #35 @@ -700,7 +690,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { Delay: 5, CanonHeight: 30, CanonHash: hashC, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } // add some pending standard changes all on the same fork @@ -713,8 +703,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { err = authorities.addPendingChange(changeC, staticIsDescendentOf[string](true)) require.NoError(t, err) - finalisedKind2 := Best[uint]{31} - delayKindFinalized2 := newDelayKind[uint](finalisedKind2) + finalisedKind2 := delayKindBest[uint]{31} // effective at #45 changeD := PendingChange[string, uint]{ @@ -722,7 +711,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { Delay: 5, CanonHeight: 40, CanonHash: hashD, - DelayKind: delayKindFinalized2, + DelayKind: finalisedKind2, } err = authorities.addPendingChange(changeD, staticIsDescendentOf[string](true)) @@ -789,7 +778,7 @@ func TestForceChangesBlockedByStandardChanges(t *testing.T) { set: AuthoritySet[string, uint]{ CurrentAuthorities: setA, SetID: 3, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: expChanges, }, @@ -815,13 +804,12 @@ func TestNextChangeWorks(t *testing.T) { authorities := AuthoritySet[string, uint]{ CurrentAuthorities: currentAuthorities, SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} // We have three pending changes with 2 possible roots that are enacted // immediately on finality (i.e. standard changes). @@ -830,7 +818,7 @@ func TestNextChangeWorks(t *testing.T) { Delay: 0, CanonHeight: 5, CanonHash: "hash_a0", - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeA1 := PendingChange[string, uint]{ @@ -838,7 +826,7 @@ func TestNextChangeWorks(t *testing.T) { Delay: 0, CanonHeight: 10, CanonHash: "hash_a1", - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } changeB := PendingChange[string, uint]{ @@ -846,7 +834,7 @@ func TestNextChangeWorks(t *testing.T) { Delay: 0, CanonHeight: 4, CanonHash: hashB, - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } // A0 (#5) <- A10 (#8) <- A1 (#10) <- best_a @@ -910,14 +898,13 @@ func TestNextChangeWorks(t *testing.T) { require.Nil(t, c) // we a forced HashNumber at A10 (#8) - finalisedKind2 := Best[uint]{0} - delayKindFinalized2 := newDelayKind[uint](finalisedKind2) + finalisedKind2 := delayKindBest[uint]{0} changeA10 := PendingChange[string, uint]{ NextAuthorities: currentAuthorities, Delay: 0, CanonHeight: 8, CanonHash: "hash_a10", - DelayKind: delayKindFinalized2, + DelayKind: finalisedKind2, } err = authorities.addPendingChange(changeA10, staticIsDescendentOf[string](false)) @@ -941,7 +928,7 @@ func TestMaintainsAuthorityListInvariants(t *testing.T) { // []Authority[dummyAuthID]{}, nil, 0, - NewChangeTree[string, uint](), + forktree.NewForkTree[string, uint, PendingChange[string, uint]](), nil, nil, ) @@ -964,7 +951,7 @@ func TestMaintainsAuthorityListInvariants(t *testing.T) { _, err = NewAuthoritySet[string, uint]( invalidAuthoritiesWeight, 0, - NewChangeTree[string, uint](), + forktree.NewForkTree[string, uint, PendingChange[string, uint]](), nil, nil, ) @@ -978,29 +965,27 @@ func TestMaintainsAuthorityListInvariants(t *testing.T) { }) require.NoError(t, err) - finalisedKind := Finalized{} - delayKindFinalized := newDelayKind[uint](finalisedKind) + finalisedKind := delayKindFinalized{} invalidChangeEmptyAuthorities := PendingChange[string, uint]{ NextAuthorities: nil, Delay: 10, CanonHeight: 5, CanonHash: "", - DelayKind: delayKindFinalized, + DelayKind: finalisedKind, } // pending HashNumber contains an empty authority set err = authoritySet.addPendingChange(invalidChangeEmptyAuthorities, staticIsDescendentOf[string](false)) require.ErrorIs(t, err, errInvalidAuthoritySet) - delayKind := Best[uint]{0} - delayKindBest := newDelayKind[uint](delayKind) + delayKind := delayKindBest[uint]{0} invalidChangeAuthoritiesWeight := PendingChange[string, uint]{ NextAuthorities: invalidAuthoritiesWeight, Delay: 10, CanonHeight: 5, CanonHash: "", - DelayKind: delayKindBest, + DelayKind: delayKind, } // pending HashNumber contains an authority set @@ -1020,7 +1005,7 @@ func TestCleanUpStaleForcedChangesWhenApplyingStandardChange(t *testing.T) { authorities := AuthoritySet[string, uint]{ CurrentAuthorities: currentAuthorities, SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -1065,24 +1050,22 @@ func TestCleanUpStaleForcedChangesWhenApplyingStandardChange(t *testing.T) { addPendingChangeFunction := func(canonHeight uint, canonHash string, forced bool) { var change PendingChange[string, uint] if forced { - delayKind := Best[uint]{0} - delayKindBest := newDelayKind[uint](delayKind) + delayKind := delayKindBest[uint]{0} change = PendingChange[string, uint]{ NextAuthorities: currentAuthorities, Delay: 0, CanonHeight: canonHeight, CanonHash: canonHash, - DelayKind: delayKindBest, + DelayKind: delayKind, } } else { - delayKind := Finalized{} - delayKindFinalized := newDelayKind[uint](delayKind) + delayKind := delayKindFinalized{} change = PendingChange[string, uint]{ NextAuthorities: currentAuthorities, Delay: 0, CanonHeight: canonHeight, CanonHash: canonHash, - DelayKind: delayKindFinalized, + DelayKind: delayKind, } } @@ -1126,7 +1109,7 @@ func TestCleanUpStaleForcedChangesWhenApplyingStandardChangeAlternateCase(t *tes authorities := AuthoritySet[string, uint]{ CurrentAuthorities: currentAuthorities, SetID: 0, - PendingStandardChanges: NewChangeTree[string, uint](), + PendingStandardChanges: forktree.NewForkTree[string, uint, PendingChange[string, uint]](), PendingForcedChanges: []PendingChange[string, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -1171,24 +1154,22 @@ func TestCleanUpStaleForcedChangesWhenApplyingStandardChangeAlternateCase(t *tes addPendingChangeFunction := func(canonHeight uint, canonHash string, forced bool) { var change PendingChange[string, uint] if forced { - delayKind := Best[uint]{0} - delayKindBest := newDelayKind[uint](delayKind) + delayKind := delayKindBest[uint]{0} change = PendingChange[string, uint]{ NextAuthorities: currentAuthorities, Delay: 0, CanonHeight: canonHeight, CanonHash: canonHash, - DelayKind: delayKindBest, + DelayKind: delayKind, } } else { - delayKind := Finalized{} - delayKindFinalized := newDelayKind[uint](delayKind) + delayKind := delayKindFinalized{} change = PendingChange[string, uint]{ NextAuthorities: currentAuthorities, Delay: 0, CanonHeight: canonHeight, CanonHash: canonHash, - DelayKind: delayKindFinalized, + DelayKind: delayKind, } } diff --git a/internal/client/consensus/grandpa/aux_schema.go b/internal/client/consensus/grandpa/aux_schema.go index 2d0114e718..d7f2059296 100644 --- a/internal/client/consensus/grandpa/aux_schema.go +++ b/internal/client/consensus/grandpa/aux_schema.go @@ -4,9 +4,12 @@ package grandpa import ( + "errors" "fmt" "github.com/ChainSafe/gossamer/internal/client/api" + shareddata "github.com/ChainSafe/gossamer/internal/client/consensus/common/shared-data" + primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" "github.com/ChainSafe/gossamer/internal/primitives/runtime" grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "github.com/ChainSafe/gossamer/pkg/scale" @@ -17,10 +20,105 @@ var ( concludedRounds = []byte("grandpa_concluded_rounds") authoritySetKey = []byte("grandpa_voters") bestJustification = []byte("grandpa_best_justification") + + errValueNotFound = errors.New("value not found") ) type writeAux func(insertions []api.KeyValue) error +type getGenesisAuthorities func() (primitives.AuthorityList, error) + +// / Persistent data kept between runs. +type persistentData[H runtime.Hash, N runtime.Number] struct { + authoritySet *SharedAuthoritySet[H, N] + setState *SharedVoterSetState[H, N] +} + +func loadDecoded[T any](store api.AuxStore, key []byte) (*T, error) { + encodedValue, err := store.GetAux(key) + if err != nil { + return nil, err + } + if encodedValue == nil { + return nil, nil + } + + var dst T + err = scale.Unmarshal(encodedValue, &dst) + if err != nil { + return nil, err + } + return &dst, nil +} + +func loadPersistent[H runtime.Hash, N runtime.Number]( + store api.AuxStore, + genesisHash H, + genesisNumber N, + genesisAuths getGenesisAuthorities, +) (*persistentData[H, N], error) { + genesis := grandpa.HashNumber[H, N]{Hash: genesisHash, Number: genesisNumber} + makeGenesisRound := grandpa.NewRoundState[H, N] + + authSet, err := loadDecoded[AuthoritySet[H, N]](store, authoritySetKey) + if authSet != nil { + var setState voterSetState[H, N] + state, err := loadDecoded[voterSetStateVDT[H, N]](store, setStateKey) + if err != nil { + return nil, err + } + + if state != nil && state.inner != nil { + setState = state.inner + } else { + state := makeGenesisRound(genesis) + if state.PrevoteGHOST == nil { + panic("state is for completed round; completed rounds must have a prevote ghost; qed.") + } + base := state.PrevoteGHOST + setState = newVoterSetStateLive(primitives.SetID(authSet.SetID), *authSet, *base) + } + + return &persistentData[H, N]{ + authoritySet: &SharedAuthoritySet[H, N]{inner: *shareddata.NewSharedData(*authSet)}, + setState: &SharedVoterSetState[H, N]{inner: setState}, + }, nil + } + + logger.Info("👴 Loading GRANDPA authority set from genesis on what appears to be first startup") + genesisAuthorities, err := genesisAuths() + if err != nil { + return nil, err + } + genesisSet, err := NewGenesisAuthoritySet[H, N](genesisAuthorities) + if err != nil { + panic("genesis authorities is non-empty; all weights are non-zero; qed.") + } + + state := makeGenesisRound(genesis) + base := state.PrevoteGHOST + if base == nil { + panic("state is for completed round; completed rounds must have a prevote ghost; qed.") + } + + genesisState := newVoterSetStateLive(0, *genesisSet, *base) + genesisStateVDT := newVoterSetStateVDT[H, N]() + genesisStateVDT.inner = genesisState + insert := []api.KeyValue{ + {Key: authoritySetKey, Value: scale.MustMarshal(*genesisSet)}, + {Key: setStateKey, Value: scale.MustMarshal(genesisStateVDT)}, + } + err = store.InsertAux(insert, nil) + if err != nil { + return nil, err + } + + return &persistentData[H, N]{ + authoritySet: &SharedAuthoritySet[H, N]{inner: *shareddata.NewSharedData(*genesisSet)}, + setState: &SharedVoterSetState[H, N]{inner: genesisState}, + }, nil +} + // updateAuthoritySet Update the authority set on disk after a change. // // If there has just been a handoff, pass a newSet parameter that describes the handoff. set in all cases should diff --git a/internal/client/consensus/grandpa/aux_schema_test.go b/internal/client/consensus/grandpa/aux_schema_test.go index f837f23e04..952c572dbb 100644 --- a/internal/client/consensus/grandpa/aux_schema_test.go +++ b/internal/client/consensus/grandpa/aux_schema_test.go @@ -9,6 +9,7 @@ import ( "github.com/ChainSafe/gossamer/internal/client/api" primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" + forktree "github.com/ChainSafe/gossamer/internal/utils/fork-tree" grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/require" @@ -106,7 +107,7 @@ func TestUpdateAuthoritySet(t *testing.T) { store := newDummyStore() authorities := AuthoritySet[TestString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[TestString, uint](), + PendingStandardChanges: forktree.NewForkTree[TestString, uint, PendingChange[TestString, uint]](), } err := updateAuthoritySet[TestString, uint](authorities, nil, write(store)) @@ -117,7 +118,7 @@ func TestUpdateAuthoritySet(t *testing.T) { require.NotNil(t, encData) newAuthorities := AuthoritySet[TestString, uint]{ - PendingStandardChanges: NewChangeTree[TestString, uint](), + PendingStandardChanges: forktree.NewForkTree[TestString, uint, PendingChange[TestString, uint]](), } err = scale.Unmarshal(encData, &newAuthorities) require.NoError(t, err) @@ -127,7 +128,7 @@ func TestUpdateAuthoritySet(t *testing.T) { store = newDummyStore() authorities = AuthoritySet[TestString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[TestString, uint](), + PendingStandardChanges: forktree.NewForkTree[TestString, uint, PendingChange[TestString, uint]](), } newAuthSet := &newAuthoritySet[TestString, uint]{ @@ -143,7 +144,7 @@ func TestUpdateAuthoritySet(t *testing.T) { require.NotNil(t, encData) newAuthorities = AuthoritySet[TestString, uint]{ - PendingStandardChanges: NewChangeTree[TestString, uint](), + PendingStandardChanges: forktree.NewForkTree[TestString, uint, PendingChange[TestString, uint]](), } err = scale.Unmarshal(encData, &newAuthorities) require.NoError(t, err) @@ -175,7 +176,7 @@ func TestWriteVoterSetState(t *testing.T) { authorities := AuthoritySet[TestString, uint]{ CurrentAuthorities: primitives.AuthorityList{}, SetID: 1, - PendingStandardChanges: NewChangeTree[TestString, uint](), + PendingStandardChanges: forktree.NewForkTree[TestString, uint, PendingChange[TestString, uint]](), PendingForcedChanges: []PendingChange[TestString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -226,7 +227,7 @@ func TestWriteConcludedRound(t *testing.T) { authorities := AuthoritySet[TestString, uint]{ CurrentAuthorities: primitives.AuthorityList{}, SetID: 1, - PendingStandardChanges: NewChangeTree[TestString, uint](), + PendingStandardChanges: forktree.NewForkTree[TestString, uint, PendingChange[TestString, uint]](), PendingForcedChanges: []PendingChange[TestString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } diff --git a/internal/client/consensus/grandpa/change_tree.go b/internal/client/consensus/grandpa/change_tree.go index 11cab30519..bb4eafa97f 100644 --- a/internal/client/consensus/grandpa/change_tree.go +++ b/internal/client/consensus/grandpa/change_tree.go @@ -6,6 +6,7 @@ package grandpa import ( "errors" "fmt" + "math" "github.com/ChainSafe/gossamer/pkg/scale" "golang.org/x/exp/constraints" @@ -353,10 +354,12 @@ func (ct *ChangeTree[H, N]) FinalizeWithDescendentIf( } } -func (pcn *PendingChangeNode[H, N]) importNode(hash H, +func (pcn *PendingChangeNode[H, N]) importNode( + hash H, number N, change PendingChange[H, N], - isDescendentOf IsDescendentOf[H]) (bool, error) { + isDescendentOf IsDescendentOf[H], +) (bool, error) { announcingHash := pcn.Change.CanonHash if hash == announcingHash { return false, fmt.Errorf("%w: %v", errDuplicateHashes, hash) @@ -477,3 +480,66 @@ func (ct *ChangeTree[H, N]) swapRemove(roots []*PendingChangeNode[H, N], index N func (_ *ChangeTree[H, N]) drainFilter() { //nolint //skipcq: SCC-U1000 //skipcq: RVV-B0013 // TODO implement } + +// / Map fork tree into values of new types. +// / +// / Tree traversal technique (e.g. BFS vs DFS) is left as not specified and +// / may be subject to change in the future. In other words, your predicates +// / should not rely on the observed traversal technique currently in use. +func (ct *ChangeTree[H, N]) Map(f func(h H, n N, v PendingChange[H, N])) { + // let mut queue: Vec<_> = + // self.roots.into_iter().rev().map(|node| (usize::MAX, node)).collect(); + // let mut next_queue = Vec::new(); + // let mut output = Vec::new(); + type queueItem struct { + parentIndex uint + node PendingChangeNode[H, N] + } + queue := make([]queueItem, 0) + for i := len(ct.TreeRoots) - 1; i >= 0; i-- { + node := ct.TreeRoots[i] + queue = append(queue, queueItem{ + parentIndex: math.MaxUint, + node: *node, + }) + } + nextQueue := make([]queueItem, 0) + output := make([]queueItem, 0) + + _, _ = nextQueue, output + + // while !queue.is_empty() { + for len(queue) > 0 { + // for (parent_index, node) in queue.drain(..) { + for _, item := range queue { + // let new_data = f(&node.hash, &node.number, node.data); + // let new_node = Node { + // hash: node.hash, + // number: node.number, + // data: new_data, + // children: Vec::with_capacity(node.children.len()), + // }; + // newData := f(&item.node.) + _ = item + // let node_id = output.len(); + // output.push((parent_index, new_node)); + + // for child in node.children.into_iter().rev() { + // next_queue.push((node_id, child)); + } + } + + // std::mem::swap(&mut queue, &mut next_queue); + // } + + // let mut roots = Vec::new(); + // while let Some((parent_index, new_node)) = output.pop() { + // if parent_index == usize::MAX { + // roots.push(new_node); + // } else { + // output[parent_index].1.children.push(new_node); + // } + // } + + // ForkTree { roots, best_finalized_number: self.best_finalized_number } +} diff --git a/internal/client/consensus/grandpa/communication.go b/internal/client/consensus/grandpa/communication.go index 620975ad0b..535cd4575b 100644 --- a/internal/client/consensus/grandpa/communication.go +++ b/internal/client/consensus/grandpa/communication.go @@ -188,7 +188,7 @@ func newNetworkBridge[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H] // Note the beginning of a new round to the gossipValidator. func (nb *networkBridge[H, N, Hasher]) noteRound( - round Round, setID SetID, voters *grandpa.VoterSet[primitives.AuthorityID], + round Round, setID SetID, voters grandpa.VoterSet[primitives.AuthorityID], ) { authorities := make([]primitives.AuthorityID, voters.Len()) for i, ivi := range voters.Voters() { @@ -224,7 +224,7 @@ func (nb *networkBridge[H, N, Hasher]) roundCommunication( keystore *localIDKeystore, round Round, setID SetID, - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], hasVoted hasVoted[H, N], ) (chan primitives.SignedMessage[H, N], outgoingMessages[H, N, Hasher]) { nb.noteRound(round, setID, voters) @@ -316,7 +316,7 @@ func (nb *networkBridge[H, N, Hasher]) roundCommunication( // Set up the global communication streams. func (nb *networkBridge[H, N, Hasher]) globalCommunication( setID SetID, - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], isVoter bool, ) (chan communicationIn[H, N], commitsOut[H, N, Hasher]) { authorities := make([]primitives.AuthorityID, voters.Len()) @@ -385,7 +385,7 @@ func (nb *networkBridge[H, N, Hasher]) poll() error { func incomingGlobal[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( gossipEngine *gossip.GossipEngine[H, N, Hasher], topic H, - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], validator *gossipValidator[H, N, Hasher], neighborSender neighbourPacketSender[N], // TODO: telemetry @@ -395,7 +395,7 @@ func incomingGlobal[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( notification *gossip.TopicNotification, gossipEngine *gossip.GossipEngine[H, N, Hasher], gossipValidator *gossipValidator[H, N, Hasher], - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], ) communicationIn[H, N] { cost := checkCompactCommit[H, N, Hasher]( @@ -451,7 +451,7 @@ func incomingGlobal[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( notification *gossip.TopicNotification, gossipEngine *gossip.GossipEngine[H, N, Hasher], gossipValidator *gossipValidator[H, N, Hasher], - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], ) communicationIn[H, N] { cost := checkCatchUp[H, N, Hasher](msg.Message, voters, msg.SetID) if cost != nil { @@ -594,7 +594,7 @@ func (om *outgoingMessages[H, N, Hasher]) preSend(msg grandpa.Message[H, N]) err // checks a compact commit. returns the cost associated with processing it if the commit was bad. func checkCompactCommit[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( msg primitives.CompactCommit[H, N], - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], round Round, setID SetID, // TODO: telemetry @@ -647,7 +647,7 @@ func checkCompactCommit[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[ // checks a catch up. returns the cost associated with processing it if the catch up was bad. func checkCatchUp[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( msg primitives.CatchUp[H, N], - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], setID SetID, // TODO: telemetry ) *network.ReputationChange { @@ -657,7 +657,7 @@ func checkCatchUp[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( // check total weight is not out of range for a set of votes. var checkWeight = func( - voters *grandpa.VoterSet[primitives.AuthorityID], + voters grandpa.VoterSet[primitives.AuthorityID], votes []primitives.AuthorityID, fullThreshold grandpa.VoterWeight, ) *network.ReputationChange { @@ -794,9 +794,11 @@ func newCommitsOut[H runtime.Hash, N runtime.Number, Hasher runtime.Hasher[H]]( } func (co *commitsOut[H, N, Hasher]) preSend( //nolint: unused - round Round, - commit primitives.Commit[H, N], + out grandpa.CommunicationOut[H, N, primitives.AuthoritySignature, primitives.AuthorityID], ) error { + nc := out.(grandpa.CommunicationOutCommit[H, N, primitives.AuthoritySignature, primitives.AuthorityID]) + commit := nc.Commit + round := Round(nc.Number) if !co.isVoter { return nil } diff --git a/internal/client/consensus/grandpa/communication_test.go b/internal/client/consensus/grandpa/communication_test.go index b8a25a47cd..17c5a56559 100644 --- a/internal/client/consensus/grandpa/communication_test.go +++ b/internal/client/consensus/grandpa/communication_test.go @@ -242,6 +242,7 @@ func Test_networkBridge(t *testing.T) { private := []ed25519.Keyring{ed25519.Alice, ed25519.Bob, ed25519.Charlie} public := makeIDs(private) voterSet := grandpa.NewVoterSet(public) + require.NotNil(t, voterSet) round := Round(1) setID := SetID(1) @@ -299,7 +300,7 @@ func Test_networkBridge(t *testing.T) { tester.gossipValidator.NewPeer(NoopContext{}, id, role.ObservedRoleFull) // start round, dispatch commit, and wait for broadcast. - commitsIn, commitsOut := tester.networkBridge.globalCommunication(setID, voterSet, false) + commitsIn, commitsOut := tester.networkBridge.globalCommunication(setID, *voterSet, false) _ = commitsOut { @@ -461,7 +462,7 @@ func Test_networkBridge(t *testing.T) { tester.gossipValidator.NewPeer(NoopContext{}, id, role.ObservedRoleFull) // start round, dispatch commit, and wait for broadcast. - commitsIn, _ := tester.networkBridge.globalCommunication(setID, voterSet, false) + commitsIn, _ := tester.networkBridge.globalCommunication(setID, *voterSet, false) { action, _, _ := tester.gossipValidator.doValidate(id, encodedCommit) diff --git a/internal/client/consensus/grandpa/environment.go b/internal/client/consensus/grandpa/environment.go index 7bf59e22ca..011a6e72ff 100644 --- a/internal/client/consensus/grandpa/environment.go +++ b/internal/client/consensus/grandpa/environment.go @@ -13,8 +13,8 @@ import ( "github.com/ChainSafe/gossamer/internal/client/api" "github.com/ChainSafe/gossamer/internal/client/api/utils" - "github.com/ChainSafe/gossamer/internal/client/consensus/common" "github.com/ChainSafe/gossamer/internal/primitives/blockchain" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" "github.com/ChainSafe/gossamer/internal/primitives/runtime" grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" @@ -71,12 +71,12 @@ func newCompletedRounds[H runtime.Hash, N runtime.Number]( } // Get the set-id and voter set of the completed rounds. -func (cr *completedRounds[H, N]) setInfo() (primitives.SetID, []primitives.AuthorityID) { +func (cr completedRounds[H, N]) setInfo() (primitives.SetID, []primitives.AuthorityID) { return cr.SetId, cr.Voters } // Iterate over all completed rounds. -func (cr *completedRounds[H, N]) iter() []completedRound[H, N] { +func (cr completedRounds[H, N]) iter() []completedRound[H, N] { var reversed []completedRound[H, N] for i := len(cr.Rounds) - 1; i >= 0; i-- { reversed = append(reversed, cr.Rounds[i]) @@ -85,7 +85,7 @@ func (cr *completedRounds[H, N]) iter() []completedRound[H, N] { } // Returns the last (latest) completed round -func (cr *completedRounds[H, N]) last() completedRound[H, N] { +func (cr completedRounds[H, N]) last() completedRound[H, N] { if len(cr.Rounds) == 0 { panic("inner is never empty; always contains at least genesis; qed") } @@ -643,13 +643,13 @@ type environment[ SelectChain common.SelectChain[H, N, Header] Voters grandpa.VoterSet[primitives.AuthorityID] Config Config - AuthoritySet SharedAuthoritySet[H, N] - Network networkBridge[H, N, Hasher] + AuthoritySet *SharedAuthoritySet[H, N] + Network *networkBridge[H, N, Hasher] SetID SetID - VoterSetState SharedVoterSetState[H, N] + VoterSetState *SharedVoterSetState[H, N] VotingRule VotingRule[H, N, Header] // TODO: metrics - JustificationSender *GrandpaJustificationSender[H, N, Header] + JustificationSender *GrandpaJustificationSender[H, N, Header] // meant to be optional // TODO: telemetry } @@ -694,9 +694,10 @@ func (e *environment[H, N, Hasher, Header, E]) reportEquivocation( bestBlockHash := info.BestHash bestBlockNumber := info.BestNumber - e.AuthoritySet.mtx.Lock() - defer e.AuthoritySet.mtx.Unlock() - authoritySet := e.AuthoritySet.inner + // e.AuthoritySet.mtx.Lock() + // defer e.AuthoritySet.mtx.Unlock() + authoritySet, unlock := e.AuthoritySet.inner.DataMut() + defer unlock() // block hash and number of the next pending authority set change in the given best chain. nextChange, err := authoritySet.nextChange(bestBlockHash, isDescendentOf) @@ -732,7 +733,7 @@ func (e *environment[H, N, Hasher, Header, E]) reportEquivocation( currentSetLatestHash = bestBlockHash } - runtimeAPI := e.Client.RuntimeApi() + runtimeAPI := e.Client.RuntimeAPI() // generate key ownership proof at that block keyOwnerProof := runtimeAPI.GenerateKeyOwnershipProof( @@ -823,7 +824,7 @@ func (e *environment[H, N, Hasher, Header, E]) BestChainContaining( // NOTE: when we finalize an authority set change through the sync protocol the voter is signalled asynchronously // therefore the voter could still vote in the next round before activating the new set. the [AuthoritySet] is // updated immediately thus we restrict the voter based on that. - if e.SetID != SetID(e.AuthoritySet.inner.SetID) { + if e.SetID != SetID(e.AuthoritySet.SetID()) { ch <- grandpa.BestChainOutput[H, N]{ Value: nil, Error: nil, @@ -833,7 +834,7 @@ func (e *environment[H, N, Hasher, Header, E]) BestChainContaining( } go func() { - value, err := bestChainContaining(block, e.Client, &e.AuthoritySet, e.SelectChain, e.VotingRule) + value, err := bestChainContaining(block, e.Client, e.AuthoritySet, e.SelectChain, e.VotingRule) ch <- grandpa.BestChainOutput[H, N]{ Value: value, Error: err, @@ -849,7 +850,7 @@ func (e *environment[H, N, Hasher, Header, E]) RoundData( prevoteTimer := time.NewTimer(e.Config.GossipDuration * 2) precommitTimer := time.NewTimer(e.Config.GossipDuration * 4) - localID := localAuthorityID(e.Voters, &e.Config.KeyStore) + localID := localAuthorityID(e.Voters, e.Config.KeyStore) var hasVoted hasVoted[H, N] hv := e.VoterSetState.hasVoted(primitives.RoundNumber(round)) @@ -884,7 +885,7 @@ func (e *environment[H, N, Hasher, Header, E]) RoundData( } } - in, out := e.Network.roundCommunication(keystore, Round(round), e.SetID, &e.Voters, hasVoted) + in, out := e.Network.roundCommunication(keystore, Round(round), e.SetID, e.Voters, hasVoted) convertedIn := make(chan signedMessage[H, N]) go func() { @@ -896,7 +897,7 @@ func (e *environment[H, N, Hasher, Header, E]) RoundData( // schedule incoming messages from the network to be held until corresponding blocks are imported. incoming := newUntilVoteTargetImported( e.Client.RegisterImportNotificationStream(), - &e.Network, + e.Network, e.Client, convertedIn, "round", @@ -904,7 +905,8 @@ func (e *environment[H, N, Hasher, Header, E]) RoundData( convertedOut := make(chan grandpa.SignedMessageError[H, N, primitives.AuthoritySignature, primitives.AuthorityID]) go func() { - for signed := range incoming.Chan() { + for be := range incoming.Chan() { + signed := be.Blocked convertedOut <- grandpa.SignedMessageError[H, N, primitives.AuthoritySignature, primitives.AuthorityID]{ SignedMessage: signed.SignedMessage.SignedMessage, } @@ -1211,7 +1213,7 @@ func (e *environment[H, N, Hasher, Header, E]) FinalizeBlock( ) error { return finalizeBlock( e.Client, - &e.AuthoritySet, + e.AuthoritySet, &e.Config.JustificationGenerationPeriod, hash, number, @@ -1448,7 +1450,7 @@ func finalizeBlock[ E runtime.Extrinsic, ]( client ClientForGrandpa[H, N, Hasher, Header, E], - authoritySet *SharedAuthoritySet[H, N], + sharedAuthoritySet *SharedAuthoritySet[H, N], justificationGenerationPeriod *uint32, hash H, number N, @@ -1459,8 +1461,10 @@ func finalizeBlock[ // NOTE: lock must be held through writing to DB to avoid race. this lock also implicitly synchronises the check // for last finalized number below. - authoritySet.mtx.Lock() - defer authoritySet.mtx.Unlock() + // authoritySet.mtx.Lock() + // defer authoritySet.mtx.Unlock() + authoritySet, unlock := sharedAuthoritySet.inner.DataMut() + defer unlock() status := client.Info() @@ -1483,7 +1487,7 @@ func finalizeBlock[ } // TODO: do I need to clone this? - oldAuthoritySet := authoritySet.inner + oldAuthoritySet := authoritySet.Clone() var vc voterCommand // closure specific variable checked after LockImportRun _, err := client.LockImportRun(func(importOp *api.ClientImportOperation[H, Hasher, N, Header, E]) (any, error) { @@ -1575,7 +1579,7 @@ func finalizeBlock[ canonHash := status.NewSetBlock.Hash canonNumber := status.NewSetBlock.Number // the authority set has changed. - newID, setRef := authoritySet.Current() + newID, setRef := authoritySet.current() var level func(format string, args ...interface{}) = logger.Debugf if initialSync { @@ -1598,7 +1602,7 @@ func finalizeBlock[ } if status.Changed { - err := updateAuthoritySet(authoritySet.inner, newAuthorities, func(insert []api.KeyValue) error { + err := updateAuthoritySet(*authoritySet, newAuthorities, func(insert []api.KeyValue) error { return api.ApplyAux(importOp, insert, nil) }) if err != nil { @@ -1610,17 +1614,21 @@ func finalizeBlock[ if newAuthorities != nil { vc = voterCommandChangeAuthorities[H, N](*newAuthorities) - return nil, err + return vc, nil } vc = nil - return nil, err + return vc, nil }) if vc != nil { - return vc + command, ok := vc.(voterCommand) + if !ok { + panic("unexpected type") + } + return command } if err != nil { - authoritySet.inner = oldAuthoritySet + *authoritySet = oldAuthoritySet return err } return nil diff --git a/internal/client/consensus/grandpa/environment_test.go b/internal/client/consensus/grandpa/environment_test.go index 5af9773ddd..ea96fb2129 100644 --- a/internal/client/consensus/grandpa/environment_test.go +++ b/internal/client/consensus/grandpa/environment_test.go @@ -10,6 +10,7 @@ import ( "github.com/ChainSafe/gossamer/internal/primitives/core/hash" "github.com/ChainSafe/gossamer/internal/primitives/runtime" "github.com/ChainSafe/gossamer/internal/primitives/runtime/generic" + forktree "github.com/ChainSafe/gossamer/internal/utils/fork-tree" grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/require" @@ -83,7 +84,7 @@ func TestCompleteRoundEncoding(t *testing.T) { func TestCompletedRoundsEncoding(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -173,7 +174,7 @@ func TestCompletedRounds_Iter(t *testing.T) { func TestCompletedRounds_Last(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -204,7 +205,7 @@ func TestCompletedRounds_Last(t *testing.T) { func TestCompletedRounds_Push(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -319,7 +320,7 @@ func TestVoterSetStateEncoding(t *testing.T) { func TestVoterSetState_Live(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -337,7 +338,7 @@ func TestVoterSetState_Live(t *testing.T) { func TestVoterSetState_CompletedRounds(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -368,7 +369,7 @@ func TestVoterSetState_CompletedRounds(t *testing.T) { func TestVoterSetState_LastCompletedRound(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } @@ -410,7 +411,7 @@ func TestVoterSetState_LastCompletedRound(t *testing.T) { func TestVoterSetState_WithCurrentRound(t *testing.T) { authorities := AuthoritySet[hashString, uint]{ SetID: 1, - PendingStandardChanges: NewChangeTree[hashString, uint](), + PendingStandardChanges: forktree.NewForkTree[hashString, uint, PendingChange[hashString, uint]](), PendingForcedChanges: []PendingChange[hashString, uint]{}, AuthoritySetChanges: AuthoritySetChanges[uint]{}, } diff --git a/internal/client/consensus/grandpa/grandpa.go b/internal/client/consensus/grandpa/grandpa.go index c95d190897..1cbdfd7765 100644 --- a/internal/client/consensus/grandpa/grandpa.go +++ b/internal/client/consensus/grandpa/grandpa.go @@ -6,16 +6,20 @@ package grandpa import ( "errors" "fmt" + "sync" "time" "github.com/ChainSafe/gossamer/internal/client/api" + client_common "github.com/ChainSafe/gossamer/internal/client/consensus/common" "github.com/ChainSafe/gossamer/internal/client/keystore" "github.com/ChainSafe/gossamer/internal/client/network" "github.com/ChainSafe/gossamer/internal/client/network/role" + "github.com/ChainSafe/gossamer/internal/client/network/service" peerid "github.com/ChainSafe/gossamer/internal/client/network/types/peer-id" "github.com/ChainSafe/gossamer/internal/log" papi "github.com/ChainSafe/gossamer/internal/primitives/api" "github.com/ChainSafe/gossamer/internal/primitives/blockchain" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" "github.com/ChainSafe/gossamer/internal/primitives/core/crypto" "github.com/ChainSafe/gossamer/internal/primitives/runtime" @@ -34,6 +38,50 @@ type communicationIn[H runtime.Hash, N runtime.Number] grandpa.CommunicationIn[ type communicationOut[H runtime.Hash, N runtime.Number] grandpa.CommunicationOut[ //nolint: unused H, N, primitives.AuthoritySignature, primitives.AuthorityID] +// / Shared voter state for querying. +// +// pub struct SharedVoterState { +// inner: Arc + Sync + Send>>>>, +// } +type SharedVoterState[AuthorityID comparable] struct { + inner grandpa.VoterState[AuthorityID] + sync.RWMutex +} + +// impl SharedVoterState { +// /// Create a new empty `SharedVoterState` instance. +// pub fn empty() -> Self { +// Self { inner: Arc::new(RwLock::new(None)) } +// } + +// fn reset( +// &self, +// voter_state: Box + Sync + Send>, +// ) -> Option<()> { +// let mut shared_voter_state = self.inner.try_write_for(Duration::from_secs(1))?; + +// *shared_voter_state = Some(voter_state); +// Some(()) +// } +func (svs *SharedVoterState[AuthorityID]) reset(voterState grandpa.VoterState[AuthorityID]) error { + svs.Lock() + defer svs.Unlock() + svs.inner = voterState + return nil +} + +// /// Get the inner `VoterState` instance. +// pub fn voter_state(&self) -> Option> { +// self.inner.read().as_ref().map(|vs| vs.get()) +// } +// } + +// impl Clone for SharedVoterState { +// fn clone(&self) -> Self { +// SharedVoterState { inner: self.inner.clone() } +// } +// } + type Config struct { // The expected duration for a message to be gossiped across the network. GossipDuration time.Duration @@ -95,10 +143,10 @@ type ClientForGrandpa[ blockchain.HeaderMetadata[H, N] blockchain.HeaderBackend[H, N, Header] api.BlockchainEvents[H, N, Header] - papi.ProvideRuntimeApi[primitives.GrandpaAPI[H, N]] + papi.ProvideRuntimeAPI[primitives.GrandpaAPI[H, N]] // api.ExecutorProvider - // common.BlockImport[H, N] - // api.StorageProvider[H, N, Hasher] + client_common.BlockImport[H, N, E, Header] + api.StorageProvider[H, N, Hasher] } // Something that one can ask to do a block sync request. @@ -121,6 +169,7 @@ type newAuthoritySet[H, N any] struct { // Commands issued to the voter. type voterCommand interface { Error() string + isVoterCommand() } // Pause the voter for given reason. @@ -129,6 +178,7 @@ type voterCommandPause string //nolint: unused func (vcp voterCommandPause) Error() string { //nolint: unused return fmt.Sprintf("Pausing voter: %s", string(vcp)) } +func (vcp voterCommandPause) isVoterCommand() {} // New authorities. type voterCommandChangeAuthorities[H, N any] newAuthoritySet[H, N] @@ -136,16 +186,992 @@ type voterCommandChangeAuthorities[H, N any] newAuthoritySet[H, N] func (vcca voterCommandChangeAuthorities[H, N]) Error() string { return "Changing authorities" } +func (voterCommandChangeAuthorities[H, N]) isVoterCommand() {} + +// / Link between the block importer and the background voter. +// pub struct LinkHalf { +type LinkHalf[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + // client: Arc, + client ClientForGrandpa[H, N, Hasher, Header, E] + // select_chain: SC, + selectChain common.SelectChain[H, N, Header] + // persistent_data: PersistentData, + persistentData persistentData[H, N] + // voter_commands_rx: TracingUnboundedReceiver>>, + voterCommandsRx chan voterCommand + // justification_sender: GrandpaJustificationSender, + justificationSender GrandpaJustificationSender[H, N, Header] + // justification_stream: GrandpaJustificationStream, + justificationStream GrandpaJustificationStream[H, N, Header] + // telemetry: Option, +} + +// impl LinkHalf { +// /// Get the shared authority set. +// pub fn shared_authority_set(&self) -> &SharedAuthoritySet> { +// &self.persistent_data.authority_set +// } + +// /// Get the receiving end of justification notifications. +// pub fn justification_stream(&self) -> GrandpaJustificationStream { +// self.justification_stream.clone() +// } +// } + +// / Provider for the Grandpa authority set configured on the genesis block. +// pub trait GenesisAuthoritySetProvider { +type GenesisAuthoritySetProvider interface { + /// Get the authority set at the genesis block. + // fn get(&self) -> Result; + Get() (primitives.AuthorityList, error) +} + +// / Make block importer and link half necessary to tie the background voter +// / to it. +// / +// / The `justification_import_period` sets the minimum period on which +// / justifications will be imported. When importing a block, if it includes a +// / justification it will only be processed if it fits within this period, +// / otherwise it will be ignored (and won't be validated). This is to avoid +// / slowing down sync by a peer serving us unnecessary justifications which +// / aren't trivial to validate. +// pub fn block_import( +// +// client: Arc, +// justification_import_period: u32, +// genesis_authorities_provider: &dyn GenesisAuthoritySetProvider, +// select_chain: SC, +// telemetry: Option, +// +// ) -> Result<(GrandpaBlockImport, LinkHalf), ClientError> +// where +// +// SC: SelectChain, +// BE: Backend + 'static, +// Client: ClientForGrandpa + 'static, +// +// { +// block_import_with_authority_set_hard_forks( +// client, +// justification_import_period, +// genesis_authorities_provider, +// select_chain, +// Default::default(), +// telemetry, +// ) +// } +func BlockImport[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + client ClientForGrandpa[H, N, Hasher, Header, E], + justificationImportPeriod uint32, + genesisAuthoritySetProvider GenesisAuthoritySetProvider, + selectChain common.SelectChain[H, N, Header], + // TODO: telemetry +) (GrandpaBlockImport[H, N, Hasher, Header, E], LinkHalf[H, N, Hasher, Header, E], error) { + return blockImportWithAuthoritySetHardForks( + client, + justificationImportPeriod, + genesisAuthoritySetProvider, + selectChain, + nil, + ) +} + +// / A descriptor for an authority set hard fork. These are authority set changes +// / that are not signalled by the runtime and instead are defined off-chain +// / (hence the hard fork). +// pub struct AuthoritySetHardFork { +type AuthoritySetHardFork[H, N any] struct { + // /// The new authority set id. + // pub set_id: SetId, + SetID SetID + // /// The block hash and number at which the hard fork should be applied. + // pub block: (Block::Hash, NumberFor), + Block HashNumber[H, N] + // /// The authorities in the new set. + // pub authorities: AuthorityList, + Authorities primitives.AuthorityList + // /// The latest block number that was finalized before this authority set + // /// hard fork. When defined, the authority set change will be forced, i.e. + // /// the node won't wait for the block above to be finalized before enacting + // /// the change, and the given finalized number will be used as a base for + // /// voting. + // pub last_finalized: Option>, + LastFinalized *N +} + +// / Make block importer and link half necessary to tie the background voter to +// / it. A vector of authority set hard forks can be passed, any authority set +// / change signaled at the given block (either already signalled or in a further +// / block when importing it) will be replaced by a standard change with the +// / given static authorities. +// pub fn block_import_with_authority_set_hard_forks( +// +// client: Arc, +// justification_import_period: u32, +// genesis_authorities_provider: &dyn GenesisAuthoritySetProvider, +// select_chain: SC, +// authority_set_hard_forks: Vec>, +// telemetry: Option, +// +// ) -> Result<(GrandpaBlockImport, LinkHalf), ClientError> +// where +// +// SC: SelectChain, +// BE: Backend + 'static, +// Client: ClientForGrandpa + 'static, +// +// { +func blockImportWithAuthoritySetHardForks[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + client ClientForGrandpa[H, N, Hasher, Header, E], + justificationImportPeriod uint32, + genesisAuthoritySetProvider GenesisAuthoritySetProvider, + selectChain common.SelectChain[H, N, Header], + authoritySetHardForks []AuthoritySetHardFork[H, N], + // TODO: telemetry +) (GrandpaBlockImport[H, N, Hasher, Header, E], LinkHalf[H, N, Hasher, Header, E], error) { + // let chain_info = client.info(); + // let genesis_hash = chain_info.genesis_hash; + chainInfo := client.Info() + genesisHash := chainInfo.GenesisHash + + // let persistent_data = + // aux_schema::load_persistent(&*client, genesis_hash, >::zero(), { + // let telemetry = telemetry.clone(); + // move || { + // let authorities = genesis_authorities_provider.get()?; + // telemetry!( + // telemetry; + // CONSENSUS_DEBUG; + // "afg.loading_authorities"; + // "authorities_len" => ?authorities.len() + // ); + // Ok(authorities) + // } + // })?; + persistentData, err := loadPersistent[H, N](client, genesisHash, 0, func() (primitives.AuthorityList, error) { + authorities, err := genesisAuthoritySetProvider.Get() + if err != nil { + return nil, err + } + return authorities, nil + }) + if err != nil { + return GrandpaBlockImport[H, N, Hasher, Header, E]{}, LinkHalf[H, N, Hasher, Header, E]{}, err + } + + _ = persistentData + + // let (voter_commands_tx, voter_commands_rx) = + // tracing_unbounded("mpsc_grandpa_voter_command", 100_000); + + // let (justification_sender, justification_stream) = GrandpaJustificationStream::channel(); + + // // create pending change objects with 0 delay for each authority set hard fork. + // let authority_set_hard_forks = authority_set_hard_forks + // .into_iter() + // .map(|fork| { + // let delay_kind = if let Some(last_finalized) = fork.last_finalized { + // authorities::DelayKind::Best { median_last_finalized: last_finalized } + // } else { + // authorities::DelayKind::Finalized + // }; + + // ( + // fork.set_id, + // authorities::PendingChange { + // next_authorities: fork.authorities, + // delay: Zero::zero(), + // canon_hash: fork.block.0, + // canon_height: fork.block.1, + // delay_kind, + // }, + // ) + // }) + // .collect(); + + // Ok(( + // + // GrandpaBlockImport::new( + // client.clone(), + // justification_import_period, + // select_chain.clone(), + // persistent_data.authority_set.clone(), + // voter_commands_tx, + // authority_set_hard_forks, + // justification_sender.clone(), + // telemetry.clone(), + // ), + // LinkHalf { + // client, + // select_chain, + // persistent_data, + // voter_commands_rx, + // justification_sender, + // justification_stream, + // telemetry, + // }, + // + // )) + panic("unimpl") +} + +// fn global_communication( +// +// set_id: SetId, +// voters: &Arc>, +// client: Arc, +// network: &NetworkBridge, +// keystore: Option<&KeystorePtr>, +// metrics: Option, +// +// ) -> ( +// +// impl Stream< +// Item = Result< +// CommunicationInH, +// CommandOrError>, +// >, +// >, +// impl Sink< +// CommunicationOutH, +// Error = CommandOrError>, +// >, +// +// ) +// where +// +// BE: Backend + 'static, +// C: ClientForGrandpa + 'static, +// N: NetworkT, +// S: SyncingT, +// NumberFor: BlockNumberOps, +// +// { +func globalCommunication[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + setID primitives.SetID, + voters grandpa.VoterSet[primitives.AuthorityID], + client ClientForGrandpa[H, N, Hasher, Header, E], + network *networkBridge[H, N, Hasher], + keystore keystore.KeyStore, + // TODO: metrics +) (chan grandpa.GlobalInItem[H, N, primitives.AuthoritySignature, primitives.AuthorityID], commitsOut[H, N, Hasher]) { + // let is_voter = local_authority_id(voters, keystore).is_some(); + isVoter := localAuthorityID(voters, keystore) != nil + + // verification stream + // let (global_in, global_out) = + // network.global_communication(communication::SetId(set_id), voters.clone(), is_voter); + in, out := network.globalCommunication(SetID(setID), voters, isVoter) + + // block commit and catch up messages until relevant blocks are imported. + // let global_in = UntilGlobalMessageBlocksImported::new( + // client.import_notification_stream(), + // network.clone(), + // client.clone(), + // global_in, + // "global", + // metrics, + // ); + globalIn := newUntilGlobalMessageBlocksImported( + client.RegisterImportNotificationStream(), + network, + client, + in, + "global", + ) + + // let global_in = global_in.map_err(CommandOrError::from); + // let global_out = global_out.sink_map_err(CommandOrError::from); + mappedIn := make(chan grandpa.GlobalInItem[H, N, primitives.AuthoritySignature, primitives.AuthorityID]) + go func() { + defer close(mappedIn) + for item := range globalIn.Chan() { + mappedIn <- grandpa.GlobalInItem[H, N, primitives.AuthoritySignature, primitives.AuthorityID]{ + CommunicationIn: item.Blocked, + Error: item.Error, + } + } + }() + + // (global_in, global_out) + return mappedIn, out +} + +// / Parameters used to run Grandpa. +// pub struct GrandpaParams { +type GrandpaParams[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + /// Configuration for the GRANDPA service. + // pub config: Config, + Config Config + /// A link to the block import worker. + // pub link: LinkHalf, + LinkHalf LinkHalf[H, N, Hasher, Header, E] + /// The Network instance. + /// + /// It is assumed that this network will feed us Grandpa notifications. When using the + /// `sc_network` crate, it is assumed that the Grandpa notifications protocol has been passed + /// to the configuration of the networking. See [`grandpa_peers_set_config`]. + // pub network: N, + Network Network + /// Event stream for syncing-related events. + // pub sync: S, + Sync Syncing[H, N] + /// Handle for interacting with `Notifications`. + // pub notification_service: Box, + NotificationService service.NotificationService + /// A voting rule used to potentially restrict target votes. + // pub voting_rule: VR, + VotingRule VotingRule[H, N, Header] + /// The prometheus metrics registry. + // pub prometheus_registry: Option, + /// The voter state is exposed at an RPC endpoint. + // pub shared_voter_state: SharedVoterState, + SharedVoterState *SharedVoterState[primitives.AuthorityID] + /// TelemetryHandle instance. + // pub telemetry: Option, + /// Offchain transaction pool factory. + /// + /// This will be used to create an offchain transaction pool instance for sending an + /// equivocation report from the runtime. + // pub offchain_tx_pool_factory: OffchainTransactionPoolFactory, +} + +// / Run a GRANDPA voter as a task. Provide configuration and a link to a +// / block import worker that has already been instantiated with `block_import`. +// pub fn run_grandpa_voter( +// +// grandpa_params: GrandpaParams, +// +// ) -> sp_blockchain::Result + Send> +// where +// +// BE: Backend + 'static, +// N: NetworkT + Sync + 'static, +// S: SyncingT + Sync + 'static, +// SC: SelectChain + 'static, +// VR: VotingRule + Clone + 'static, +// NumberFor: BlockNumberOps, +// C: ClientForGrandpa + 'static, +// C::Api: GrandpaApi, +// +// { +func RunGrandpaVoter[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + grandpaParams GrandpaParams[H, N, Hasher, Header, E], +) (done chan struct{}, err error) { + var ( + config = grandpaParams.Config + link = grandpaParams.LinkHalf + network = grandpaParams.Network + sync = grandpaParams.Sync + notificationService = grandpaParams.NotificationService + votingRule = grandpaParams.VotingRule + sharedVoterState = grandpaParams.SharedVoterState + ) + // let GrandpaParams { + // mut config, + // link, + // network, + // sync, + // notification_service, + // voting_rule, + // prometheus_registry, + // shared_voter_state, + // telemetry, + // offchain_tx_pool_factory, + // } = grandpa_params; + + // // NOTE: we have recently removed `run_grandpa_observer` from the public + // // API, I felt it is easier to just ignore this field rather than removing + // // it from the config temporarily. This should be removed after #5013 is + // // fixed and we re-add the observer to the public API. + // config.observer_enabled = false; + config.ObserverEnabled = false + + // let LinkHalf { + // client, + // select_chain, + // persistent_data, + // voter_commands_rx, + // justification_sender, + // justification_stream: _, + // telemetry: _, + // } = link; + var ( + client = link.client + selectChain = link.selectChain + persistentData = link.persistentData + voterCommandsRx = link.voterCommandsRx + justificationSender = link.justificationSender + ) + + // let network = NetworkBridge::new( + // network, + // sync, + // notification_service, + // config.clone(), + // persistent_data.set_state.clone(), + // prometheus_registry.as_ref(), + // telemetry.clone(), + // ); + networkBridge := newNetworkBridge[H, N, Hasher]( + network, + sync, + notificationService, + config, + persistentData.setState, + ) + // let conf = config.clone(); + // let telemetry_task = + // if let Some(telemetry_on_connect) = telemetry.as_ref().map(|x| x.on_connect_stream()) { + // let authorities = persistent_data.authority_set.clone(); + // let telemetry = telemetry.clone(); + // let events = telemetry_on_connect.for_each(move |_| { + // let current_authorities = authorities.current_authorities(); + // let set_id = authorities.set_id(); + // let maybe_authority_id = + // local_authority_id(¤t_authorities, conf.keystore.as_ref()); + + // let authorities = + // current_authorities.iter().map(|(id, _)| id.to_string()).collect::>(); + + // let authorities = serde_json::to_string(&authorities).expect( + // "authorities is always at least an empty vector; \ + // elements are always of type string", + // ); + + // telemetry!( + // telemetry; + // CONSENSUS_INFO; + // "afg.authority_set"; + // "authority_id" => maybe_authority_id.map_or("".into(), |s| s.to_string()), + // "authority_set_id" => ?set_id, + // "authorities" => authorities, + // ); + + // future::ready(()) + // }); + // future::Either::Left(events) + // } else { + // future::Either::Right(future::pending()) + // }; + + // let voter_work = VoterWork::new( + // client, + // config, + // network, + // select_chain, + // voting_rule, + // persistent_data, + // voter_commands_rx, + // prometheus_registry, + // shared_voter_state, + // justification_sender, + // telemetry, + // offchain_tx_pool_factory, + // ); + voterWork := newVoterWork[H, N, Hasher, Header, E]( + client, + config, + networkBridge, + selectChain, + votingRule, + persistentData, + voterCommandsRx, + sharedVoterState, + justificationSender, + ) + + // let voter_work = voter_work.map(|res| match res { + // Ok(()) => error!( + // target: LOG_TARGET, + // "GRANDPA voter future has concluded naturally, this should be unreachable." + // ), + // Err(e) => error!(target: LOG_TARGET, "GRANDPA voter error: {}", e), + // }); + done = make(chan struct{}) + go func() { + defer close(done) + err := voterWork.run() + if err != nil { + logger.Errorf("GRANDPA voter error: %v", err) + return + } + logger.Error("GRANDPA voter future has concluded naturally, this should be unreachable.") + }() + + // // Make sure that `telemetry_task` doesn't accidentally finish and kill grandpa. + // let telemetry_task = telemetry_task.then(|_| future::pending::<()>()); + + // Ok(future::select(voter_work, telemetry_task).map(drop)) + return done, nil +} + +// / Future that powers the voter. +type voterWork[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + // use string for AuthorityID and AuthoritySignature + voter *grandpa.Voter[H, N, primitives.AuthoritySignature, primitives.AuthorityID] + voterErrChan <-chan error + sharedVoterState *SharedVoterState[primitives.AuthorityID] + env *environment[H, N, Hasher, Header, E] + voterCommandsRx <-chan voterCommand + network *networkBridge[H, N, Hasher] + // TODO: telemtry, metrics +} + +func newVoterWork[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + client ClientForGrandpa[H, N, Hasher, Header, E], + config Config, + network *networkBridge[H, N, Hasher], + selectChain common.SelectChain[H, N, Header], + votingRule VotingRule[H, N, Header], + persistentData persistentData[H, N], + voterCommandsRx <-chan voterCommand, + sharedVoterState *SharedVoterState[primitives.AuthorityID], + justificationSender GrandpaJustificationSender[H, N, Header], + // TODO: telemetry +) voterWork[H, N, Hasher, Header, E] { + // TODO: register to prometheus registry + + voters := persistentData.authoritySet.CurrentAuthorities() + env := environment[H, N, Hasher, Header, E]{ + Client: client, + SelectChain: selectChain, + VotingRule: votingRule, + Voters: voters, + Config: config, + Network: network, + SetID: SetID(persistentData.authoritySet.SetID()), + AuthoritySet: persistentData.authoritySet, + VoterSetState: persistentData.setState, + JustificationSender: &justificationSender, + } + + work := voterWork[H, N, Hasher, Header, E]{ + // `voter` is set to a temporary value and replaced below when + // calling `rebuild_voter`. + voter: nil, + sharedVoterState: sharedVoterState, + env: &env, + voterCommandsRx: voterCommandsRx, + network: network, + } + work.rebuildVoter() + return work +} + +// / Rebuilds the `self.voter` field using the current authority set +// / state. This method should be called when we know that the authority set +// / has changed (e.g. as signalled by a voter command). +func (vw *voterWork[H, N, Hasher, Header, E]) rebuildVoter() { + // debug!( + // target: LOG_TARGET, + // "{}: Starting new voter with set ID {}", + // self.env.config.name(), + // self.env.set_id + // ); + logger.Debugf("%s: Starting new voter with set ID %v", vw.env.Config.name(), vw.env.SetID) + + maybeAuthorityID := localAuthorityID(vw.env.Voters, vw.env.Config.KeyStore) + var authorityID string + if maybeAuthorityID != nil { + authorityID = string(maybeAuthorityID.Bytes()) + } else { + authorityID = "" + } + + // telemetry!( + // self.telemetry; + // CONSENSUS_DEBUG; + // "afg.starting_new_voter"; + // "name" => ?self.env.config.name(), + // "set_id" => ?self.env.set_id, + // "authority_id" => authority_id, + // ); + // TODO: telemetry afg.starting_new_voter + + chainInfo := vw.env.Client.Info() + + // let authorities = self.env.voters.iter().map(|(id, _)| id.to_string()).collect::>(); + + // let authorities = serde_json::to_string(&authorities).expect( + // "authorities is always at least an empty vector; elements are always of type string; qed.", + // ); + + // telemetry!( + // self.telemetry; + // CONSENSUS_INFO; + // "afg.authority_set"; + // "number" => ?chain_info.finalized_number, + // "hash" => ?chain_info.finalized_hash, + // "authority_id" => authority_id, + // "authority_set_id" => ?self.env.set_id, + // "authorities" => authorities, + // ); + // TODO: telemetry afg.authority_set + + _ = authorityID + _ = chainInfo + + vw.env.VoterSetState.innerMtx.RLock() + defer vw.env.VoterSetState.innerMtx.RUnlock() + + switch vss := vw.env.VoterSetState.inner.(type) { + case voterSetStateLive[H, N]: + // let last_finalized = (chain_info.finalized_hash, chain_info.finalized_number); + var lastFinalized = grandpa.HashNumber[H, N]{ + Hash: chainInfo.FinalizedHash, + Number: chainInfo.FinalizedNumber, + } + _ = lastFinalized + // let global_comms = global_communication( + // self.env.set_id, + // &self.env.voters, + // self.env.client.clone(), + // &self.env.network, + // self.env.config.keystore.as_ref(), + // self.metrics.as_ref().map(|m| m.until_imported.clone()), + // ); + globalIn, globalOut := globalCommunication( + primitives.SetID(vw.env.SetID), + vw.env.Voters, + vw.env.Client, + vw.env.Network, + vw.env.Config.KeyStore, + ) + + // let last_completed_round = completed_rounds.last(); + lastCompletedRound := vss.completedRounds().last() + + // let voter = voter::Voter::new( + // self.env.clone(), + // (*self.env.voters).clone(), + // global_comms, + // last_completed_round.number, + // last_completed_round.votes.clone(), + // last_completed_round.base, + // last_finalized, + // ); + votes := make([]grandpa.SignedMessage[H, N, primitives.AuthoritySignature, primitives.AuthorityID], len(lastCompletedRound.Votes)) + for i, vote := range lastCompletedRound.Votes { + votes[i] = grandpa.SignedMessage[H, N, primitives.AuthoritySignature, primitives.AuthorityID]{ + Signature: vote.Signature, + Message: vote.Message, + ID: vote.ID, + } + } + voter := grandpa.NewVoter[H, N, primitives.AuthoritySignature, primitives.AuthorityID]( + vw.env, + vw.env.Voters, + globalIn, + globalOut.preSend, + uint64(lastCompletedRound.Number), + votes, + lastCompletedRound.Base, + lastFinalized, + ) + + // // Repoint shared_voter_state so that the RPC endpoint can query the state + // if self.shared_voter_state.reset(voter.voter_state()).is_none() { + // info!( + // target: LOG_TARGET, + // "Timed out trying to update shared GRANDPA voter state. \ + // RPC endpoints may return stale data." + // ); + // } + vw.sharedVoterState.reset(voter.VoterState()) + + // self.voter = Box::pin(voter); + vw.voter = voter + errChan := make(chan error) + go func() { + err := voter.Start() + errChan <- err + close(errChan) + }() + vw.voterErrChan = errChan + case voterSetStatePaused[H, N]: + // VoterSetState::Paused { .. } => self.voter = Box::pin(future::pending()), + // TODO: don't do anything? I dunno + default: + panic("unreachable") + } +} + +// fn handle_voter_command( +// +// &mut self, +// command: VoterCommand>, +// +// ) -> Result<(), Error> { +func (vw *voterWork[H, N, Hasher, Header, E]) handleVoterCommand(command voterCommand) error { + // match command { + switch command := command.(type) { + // VoterCommand::ChangeAuthorities(new) => { + case voterCommandChangeAuthorities[H, N]: + new := command + // let voters: Vec = + // new.authorities.iter().map(move |(a, _)| format!("{}", a)).collect(); + // telemetry!( + // self.telemetry; + // CONSENSUS_INFO; + // "afg.voter_command_change_authorities"; + // "number" => ?new.canon_number, + // "hash" => ?new.canon_hash, + // "voters" => ?voters, + // "set_id" => ?new.set_id, + // ); + // TODO: telemetry + + // self.env.update_voter_set_state(|_| { + // // start the new authority set using the block where the + // // set changed (not where the signal happened!) as the base. + // let set_state = VoterSetState::live( + // new.set_id, + // &*self.env.authority_set.inner(), + // (new.canon_hash, new.canon_number), + // ); + + // aux_schema::write_voter_set_state(&*self.env.client, &set_state)?; + // Ok(Some(set_state)) + // })?; + err := vw.env.updateVoterSetState(func(voterSetState voterSetState[H, N]) (voterSetState[H, N], error) { + // start the new authority set using the block where the + // set changed (not where the signal happened!) as the base. + // vw.env.AuthoritySet.mtx.Lock() + authoritySet, unlock := vw.env.AuthoritySet.inner.DataMut() + defer unlock() + setState := newVoterSetStateLive( + primitives.SetID(vw.env.SetID), + *authoritySet, + grandpa.HashNumber[H, N]{ + Hash: command.CanonHash, + Number: command.CanonNumber, + }, + ) + // vw.env.AuthoritySet.mtx.Unlock() + err := writeVoterSetState(vw.env.Client, &setState) + if err != nil { + return nil, err + } + return setState, nil + }) + if err != nil { + return err + } + + // let voters = Arc::new(VoterSet::new(new.authorities.into_iter()).expect( + // "new authorities come from pending change; pending change comes from \ + // `AuthoritySet`; `AuthoritySet` validates authorities is non-empty and \ + // weights are non-zero; qed.", + // )); + authorites := make([]grandpa.IDWeight[primitives.AuthorityID], len(new.Authorities)) + for i, authority := range new.Authorities { + authorites[i] = grandpa.IDWeight[primitives.AuthorityID]{ + ID: authority.AuthorityID, + Weight: uint64(authority.AuthorityWeight), + } + } + voters := grandpa.NewVoterSet(authorites) + if voters == nil { + panic("new authorities come from pending change; pending change comes from AuthoritySet; AuthoritySet validates authorities is non-empty and weights are non-zero") + } + + // self.env = Arc::new(Environment { + // voters, + // set_id: new.set_id, + // voter_set_state: self.env.voter_set_state.clone(), + // client: self.env.client.clone(), + // select_chain: self.env.select_chain.clone(), + // config: self.env.config.clone(), + // authority_set: self.env.authority_set.clone(), + // network: self.env.network.clone(), + // voting_rule: self.env.voting_rule.clone(), + // metrics: self.env.metrics.clone(), + // justification_sender: self.env.justification_sender.clone(), + // telemetry: self.telemetry.clone(), + // offchain_tx_pool_factory: self.env.offchain_tx_pool_factory.clone(), + // _phantom: PhantomData, + // }); + vw.env = &environment[H, N, Hasher, Header, E]{ + Voters: *voters, + SetID: SetID(new.SetID), + VoterSetState: vw.env.VoterSetState, + Client: vw.env.Client, + SelectChain: vw.env.SelectChain, + Config: vw.env.Config, + AuthoritySet: vw.env.AuthoritySet, + Network: vw.env.Network, + VotingRule: vw.env.VotingRule, + JustificationSender: vw.env.JustificationSender, + } + + // self.rebuild_voter(); + vw.rebuildVoter() + // Ok(()) + return nil + // }, + // VoterCommand::Pause(reason) => { + case voterCommandPause: + // info!(target: LOG_TARGET, "Pausing old validator set: {}", reason); + logger.Infof("Pausing old validator set: %s", string(command)) + + // not racing because old voter is shut down. + // self.env.update_voter_set_state(|voter_set_state| { + err := vw.env.updateVoterSetState(func(voterSetState voterSetState[H, N]) (voterSetState[H, N], error) { + // let completed_rounds = voter_set_state.completed_rounds(); + // let set_state = VoterSetState::Paused { completed_rounds }; + completedRounds := voterSetState.completedRounds() + setState := voterSetStatePaused[H, N]{CompletedRounds: completedRounds} + // aux_schema::write_voter_set_state(&*self.env.client, &set_state)?; + err := writeVoterSetState(vw.env.Client, setState) + if err != nil { + return nil, err + } + // Ok(Some(set_state)) + return setState, nil + // })?; + }) + if err != nil { + return err + } + + // self.rebuild_voter(); + vw.rebuildVoter() + // Ok(()) + return nil + // }, + default: + panic("unreachable") + } +} + +// fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { +func (vw *voterWork[H, N, Hasher, Header, E]) poll() error { + // match Future::poll(Pin::new(&mut self.voter), cx) { + // Poll::Pending => {}, + // Poll::Ready(Ok(())) => { + // // voters don't conclude naturally + // return Poll::Ready(Err(Error::Safety( + // "consensus-grandpa inner voter has concluded.".into(), + // ))) + // }, + // Poll::Ready(Err(CommandOrError::Error(e))) => { + // // return inner observer error + // return Poll::Ready(Err(e)) + // }, + // Poll::Ready(Err(CommandOrError::VoterCommand(command))) => { + // // some command issued internally + // self.handle_voter_command(command)?; + // cx.waker().wake_by_ref(); + // }, + // } + select { + case err := <-vw.voterErrChan: + if err == nil { + // voters don't conclude naturally + return fmt.Errorf("consensus-grandpa inner voter has concluded: %w", ErrSafety) + } + vc, isVoterCommand := err.(voterCommand) + if !isVoterCommand { + // return inner observer error + return err + } + // some command issued internally + return vw.handleVoterCommand(vc) + default: + } + + // match Stream::poll_next(Pin::new(&mut self.voter_commands_rx), cx) { + // Poll::Pending => {}, + // Poll::Ready(None) => { + // // the `voter_commands_rx` stream should never conclude since it's never closed. + // return Poll::Ready(Err(Error::Safety("`voter_commands_rx` was closed.".into()))) + // }, + // Poll::Ready(Some(command)) => { + // // some command issued externally + // self.handle_voter_command(command)?; + // cx.waker().wake_by_ref(); + // }, + // } + select { + case vc, ok := <-vw.voterCommandsRx: + if !ok { + // the `voter_commands_rx` stream should never conclude since it's never closed. + return fmt.Errorf("`%w: voter_commands_rx` was closed", ErrSafety) + } + // some command issued externally + return vw.handleVoterCommand(vc) + default: + } + return nil +} + +func (vw *voterWork[H, N, Hasher, Header, E]) run() error { + for { + err := vw.poll() + if err != nil { + return err + } + } +} // Checks if this node has any available keys in the keystore for any authority id in the givenvoter set. Returns the // authority id for which keys are available, or nil if no keys are available. -func localAuthorityID(voters grandpa.VoterSet[primitives.AuthorityID], ks *keystore.KeyStore) *primitives.AuthorityID { +func localAuthorityID(voters grandpa.VoterSet[primitives.AuthorityID], ks keystore.KeyStore) *primitives.AuthorityID { if ks == nil { return nil } for _, voter := range voters.Voters() { - if (*ks).HasKeys([]keystore.PublicKey{{ + if ks.HasKeys([]keystore.PublicKey{{ Key: voter.ID.Bytes(), KeyTypeID: crypto.GRANDPA, }}) { diff --git a/internal/client/consensus/grandpa/grandpa_test.go b/internal/client/consensus/grandpa/grandpa_test.go new file mode 100644 index 0000000000..9ad30122b5 --- /dev/null +++ b/internal/client/consensus/grandpa/grandpa_test.go @@ -0,0 +1,514 @@ +package grandpa_test + +import ( + "math" + "sync" + "testing" + + "github.com/ChainSafe/gossamer/internal/client/consensus/grandpa" + common_sync "github.com/ChainSafe/gossamer/internal/client/network/common/sync" + "github.com/ChainSafe/gossamer/internal/client/network/test" + primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" + "github.com/ChainSafe/gossamer/internal/primitives/keyring/ed25519" + "github.com/ChainSafe/gossamer/internal/test-utils/runtime" + "github.com/ChainSafe/gossamer/internal/test-utils/runtime/client" +) + +// type TestLinkHalf = LinkHalf>; +type TestLinkHalf = grandpa.LinkHalf[runtime.Hash, runtime.BlockNumber, runtime.Hasher, runtime.Header, runtime.Extrinsic] + +// type PeerData = Mutex>; +type PeerData struct { + *TestLinkHalf + sync.Mutex +} +type GrandpaPeer = test.Peer[*PeerData, grandpa.GrandpaBlockImport[runtime.Hash, runtime.BlockNumber, runtime.Hasher, runtime.Header, runtime.Extrinsic]] + +// #[derive(Default)] +// +// struct GrandpaTestNet { +// peers: Vec, +// test_config: TestApi, +// } +type GrandpaTestNet struct { + peers []GrandpaPeer + testConfig TestAPI +} + +// impl GrandpaTestNet { +// fn new(test_config: TestApi, n_authority: usize, n_full: usize) -> Self { +// let mut net = +// GrandpaTestNet { peers: Vec::with_capacity(n_authority + n_full), test_config }; + +// for _ in 0..n_authority { +// net.add_authority_peer(); +// } + +// for _ in 0..n_full { +// net.add_full_peer(); +// } + +// net +// } +// } +func NewGrandpaTestNet(testConfig TestAPI, nAuthority, nFull int) *GrandpaTestNet { + net := &GrandpaTestNet{ + peers: make([]GrandpaPeer, 0, nAuthority+nFull), + testConfig: testConfig, + } + + for i := 0; i < nAuthority; i++ { + net.addAuthorityPeer() + } + + for i := 0; i < nFull; i++ { + net.addFullPeer() + } + + return net +} + +// impl GrandpaTestNet { +// fn add_authority_peer(&mut self) { +func (gtn *GrandpaTestNet) addAuthorityPeer() { + // self.add_full_peer_with_config(FullPeerConfig { + // notifications_protocols: vec![grandpa_protocol_name::NAME.into()], + // is_authority: true, + // ..Default::default() + // }) + // } + panic("unimplemented") +} + +// impl TestNetFactory for GrandpaTestNet { +// type Verifier = PassThroughVerifier; +// type PeerData = PeerData; +// type BlockImport = GrandpaBlockImport; + +// fn add_full_peer(&mut self) { +// self.add_full_peer_with_config(FullPeerConfig { +// notifications_protocols: vec![grandpa_protocol_name::NAME.into()], +// is_authority: false, +// ..Default::default() +// }) +// } +func (gtn *GrandpaTestNet) addFullPeer() { + // self.add_full_peer_with_config(FullPeerConfig { + // notifications_protocols: vec![grandpa_protocol_name::NAME.into()], + // is_authority: false, + // ..Default::default() + // }) + // } + panic("unimplemented") +} + +// fn make_verifier(&self, _client: PeersClient, _: &PeerData) -> Self::Verifier { +// PassThroughVerifier::new(false) // use non-instant finality. +// } + +// fn make_block_import( +// +// &self, +// client: PeersClient, +// +// ) -> (BlockImportAdapter, Option>, PeerData) { +// let (client, backend) = (client.as_client(), client.as_backend()); +// let (import, link) = block_import( +// client.clone(), +// JUSTIFICATION_IMPORT_PERIOD, +// &self.test_config, +// LongestChain::new(backend.clone()), +// None, +// ) +// .expect("Could not create block import for fresh peer."); +// let justification_import = Box::new(import.clone()); +// (BlockImportAdapter::new(import), Some(justification_import), Mutex::new(Some(link))) +// } +func (gtn *GrandpaTestNet) makeBlockImport(client test.PeersClient) { + panic("unimpl") +} + +// fn peer(&mut self, i: usize) -> &mut GrandpaPeer { +// &mut self.peers[i] +// } + +// fn peers(&self) -> &Vec { +// &self.peers +// } + +// fn peers_mut(&mut self) -> &mut Vec { +// &mut self.peers +// } + +// fn mut_peers)>(&mut self, closure: F) { +// closure(&mut self.peers); +// } +// } + +// / Add a full peer. +// fn add_full_peer_with_config(&mut self, config: FullPeerConfig) { +func (gtn *GrandpaTestNet) addFullPeerWithConfig(config test.FullPeerConfig) { + // let mut test_client_builder = match (config.blocks_pruning, config.storage_chain) { + // (Some(blocks_pruning), true) => TestClientBuilder::with_tx_storage(blocks_pruning), + // (None, true) => TestClientBuilder::with_tx_storage(u32::MAX), + // (Some(blocks_pruning), false) => TestClientBuilder::with_pruning_window(blocks_pruning), + // (None, false) => TestClientBuilder::with_default_backend(), + // }; + var testClientBuilder client.TestClientBuilder + switch { + case config.BlocksPruning != nil && config.StorageChain: + testClientBuilder = client.NewTestClientBuilderWithTxStorage(*config.BlocksPruning) + case config.BlocksPruning == nil && config.StorageChain: + testClientBuilder = client.NewTestClientBuilderWithTxStorage(math.MaxUint32) + case config.BlocksPruning != nil && !config.StorageChain: + testClientBuilder = client.NewTestClientBuilderWithPruningWindow(*config.BlocksPruning) + case config.BlocksPruning == nil && !config.StorageChain: + testClientBuilder = client.NewTestClientBuilderWithDefaultBackend() + default: + panic("unreachable") + } + // if let Some(storage) = config.extra_storage { + // let genesis_extra_storage = test_client_builder.genesis_init_mut().extra_storage(); + // *genesis_extra_storage = storage; + // } + if config.ExtraStorage != nil { + genesisExtraStorage := testClientBuilder.GenesisInitMut().ExtraStorage() + *genesisExtraStorage = *config.ExtraStorage + } + + // if !config.force_genesis && + // matches!(config.sync_mode, SyncMode::LightState { .. } | SyncMode::Warp) + // { + // test_client_builder = test_client_builder.set_no_genesis(); + // } + if !config.ForceGenesis { + switch config.SyncMode.(type) { + case common_sync.SyncModeLightState, common_sync.SyncModeWarp: + testClientBuilder.SetNoGenesis() + } + } + // let backend = test_client_builder.backend(); + // let (c, longest_chain) = test_client_builder.build_with_longest_chain(); + // let client = Arc::new(c); + backend := testClientBuilder.Backend() + c, longestChain := testClientBuilder.BuildWithLongestChain() + + _, _, _ = backend, c, longestChain + + // let (block_import, justification_import, data) = self + // .make_block_import(PeersClient { client: client.clone(), backend: backend.clone() }); + + // let verifier = self + // .make_verifier(PeersClient { client: client.clone(), backend: backend.clone() }, &data); + // let verifier = VerifierAdapter::new(verifier); + + // let import_queue = Box::new(BasicQueue::new( + // verifier.clone(), + // Box::new(block_import.clone()), + // justification_import, + // &sp_core::testing::TaskExecutor::new(), + // None, + // )); + + // let listen_addr = build_multiaddr![Memory(rand::random::())]; + + // let mut network_config = + // NetworkConfiguration::new("test-node", "test-client", Default::default(), None); + // network_config.sync_mode = config.sync_mode; + // network_config.transport = TransportConfig::MemoryOnly; + // network_config.listen_addresses = vec![listen_addr.clone()]; + // network_config.allow_non_globals_in_dht = true; + + // let (notif_configs, notif_handles): (Vec<_>, Vec<_>) = config + // .notifications_protocols + // .into_iter() + // .map(|p| { + // let (config, handle) = NonDefaultSetConfig::new( + // p.clone(), + // Vec::new(), + // 1024 * 1024, + // None, + // Default::default(), + // ); + + // (config, (p, handle)) + // }) + // .unzip(); + + // if let Some(connect_to) = config.connect_to_peers { + // let addrs = connect_to + // .iter() + // .map(|v| { + // let peer_id = self.peer(*v).network_service().local_peer_id(); + // let multiaddr = self.peer(*v).listen_addr.clone(); + // MultiaddrWithPeerId { peer_id, multiaddr } + // }) + // .collect(); + // network_config.default_peers_set.reserved_nodes = addrs; + // network_config.default_peers_set.non_reserved_mode = NonReservedPeerMode::Deny; + // } + // let mut full_net_config = FullNetworkConfiguration::new(&network_config, None); + + // let protocol_id = ProtocolId::from("test-protocol-name"); + + // let fork_id = Some(String::from("test-fork-id")); + + // let chain_sync_network_provider = NetworkServiceProvider::new(); + // let chain_sync_network_handle = chain_sync_network_provider.handle(); + // let mut block_relay_params = BlockRequestHandler::new::>( + // chain_sync_network_handle.clone(), + // &protocol_id, + // None, + // client.clone(), + // 50, + // ); + // self.spawn_task(Box::pin(async move { + // block_relay_params.server.run().await; + // })); + + // let state_request_protocol_config = { + // let (handler, protocol_config) = StateRequestHandler::new::>( + // &protocol_id, + // None, + // client.clone(), + // 50, + // ); + // self.spawn_task(handler.run().boxed()); + // protocol_config + // }; + + // let light_client_request_protocol_config = + // { + // let (handler, protocol_config) = LightClientRequestHandler::new::< + // NetworkWorker<_, _>, + // >(&protocol_id, None, client.clone()); + // self.spawn_task(handler.run().boxed()); + // protocol_config + // }; + + // let warp_sync = Arc::new(TestWarpSyncProvider(client.clone())); + + // let warp_sync_config = match config.target_header { + // Some(target_header) => WarpSyncConfig::WithTarget(target_header), + // _ => WarpSyncConfig::WithProvider(warp_sync.clone()), + // }; + + // let warp_protocol_config = { + // let (handler, protocol_config) = + // warp_request_handler::RequestHandler::new::<_, NetworkWorker<_, _>>( + // protocol_id.clone(), + // client + // .block_hash(0u32.into()) + // .ok() + // .flatten() + // .expect("Genesis block exists; qed"), + // None, + // warp_sync.clone(), + // ); + // self.spawn_task(handler.run().boxed()); + // protocol_config + // }; + + // let peer_store = PeerStore::new( + // network_config + // .boot_nodes + // .iter() + // .map(|bootnode| bootnode.peer_id.into()) + // .collect(), + // None, + // ); + // let peer_store_handle = Arc::new(peer_store.handle()); + // self.spawn_task(peer_store.run().boxed()); + + // let block_announce_validator = config + // .block_announce_validator + // .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); + // let metrics = as sc_network::NetworkBackend< + // Block, + // ::Hash, + // >>::register_notification_metrics(None); + + // let syncing_config = PolkadotSyncingStrategyConfig { + // mode: network_config.sync_mode, + // max_parallel_downloads: network_config.max_parallel_downloads, + // max_blocks_per_request: network_config.max_blocks_per_request, + // metrics_registry: None, + // state_request_protocol_name: state_request_protocol_config.name.clone(), + // block_downloader: block_relay_params.downloader, + // }; + // // Initialize syncing strategy. + // let syncing_strategy = Box::new( + // PolkadotSyncingStrategy::new( + // syncing_config, + // client.clone(), + // Some(warp_sync_config), + // Some(warp_protocol_config.name.clone()), + // ) + // .unwrap(), + // ); + + // let (engine, sync_service, block_announce_config) = + // sc_network_sync::engine::SyncingEngine::new( + // Roles::from(if config.is_authority { &Role::Authority } else { &Role::Full }), + // client.clone(), + // None, + // metrics, + // &full_net_config, + // protocol_id.clone(), + // fork_id.as_deref(), + // block_announce_validator, + // syncing_strategy, + // chain_sync_network_handle, + // import_queue.service(), + // peer_store_handle.clone(), + // ) + // .unwrap(); + // let sync_service = Arc::new(sync_service.clone()); + + // for config in config.request_response_protocols { + // full_net_config.add_request_response_protocol(config); + // } + // for config in [ + // block_relay_params.request_response_config, + // state_request_protocol_config, + // light_client_request_protocol_config, + // warp_protocol_config, + // ] { + // full_net_config.add_request_response_protocol(config); + // } + + // for config in notif_configs { + // full_net_config.add_notification_protocol(config); + // } + + // let genesis_hash = + // client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); + // let network = NetworkWorker::new(sc_network::config::Params { + // role: if config.is_authority { Role::Authority } else { Role::Full }, + // executor: Box::new(|f| { + // tokio::spawn(f); + // }), + // network_config: full_net_config, + // genesis_hash, + // protocol_id, + // fork_id, + // metrics_registry: None, + // block_announce_config, + // bitswap_config: None, + // notification_metrics: NotificationMetrics::new(None), + // }) + // .unwrap(); + + // trace!(target: "test_network", "Peer identifier: {}", network.service().local_peer_id()); + + // let service = network.service().clone(); + // tokio::spawn(async move { + // chain_sync_network_provider.run(service).await; + // }); + + // tokio::spawn({ + // let sync_service = sync_service.clone(); + + // async move { + // import_queue.run(sync_service.as_ref()).await; + // } + // }); + + // tokio::spawn(async move { + // engine.run().await; + // }); + + // self.mut_peers(move |peers| { + // for peer in peers.iter_mut() { + // peer.network.add_known_address( + // network.service().local_peer_id().into(), + // listen_addr.clone().into(), + // ); + // } + + // let imported_blocks_stream = Box::pin(client.import_notification_stream().fuse()); + // let finality_notification_stream = + // Box::pin(client.finality_notification_stream().fuse()); + + // peers.push(Peer { + // data, + // client: PeersClient { client: client.clone(), backend: backend.clone() }, + // select_chain: Some(longest_chain), + // backend: Some(backend), + // imported_blocks_stream, + // finality_notification_stream, + // notification_services: HashMap::from_iter(notif_handles.into_iter()), + // block_import, + // verifier, + // network, + // sync_service, + // listen_addr, + // }); + // }); +} + +// #[derive(Default, Clone)] +// pub(crate) struct TestApi { +type TestAPI struct { + // genesis_authorities: AuthorityList, + genesisAuthorities primitives.AuthorityList +} + +// impl TestApi { +// pub fn new(genesis_authorities: AuthorityList) -> Self { +// TestApi { genesis_authorities } +// } +// } +func NewTestAPI(genesisAuthorities primitives.AuthorityList) TestAPI { + return TestAPI{ + genesisAuthorities: genesisAuthorities, + } +} + +// fn make_ids(keys: &[Ed25519Keyring]) -> AuthorityList { +// keys.iter().map(|&key| key.public().into()).map(|id| (id, 1)).collect() +// } +func makeIDs(keys []ed25519.Keyring) primitives.AuthorityList { + ids := make(primitives.AuthorityList, len(keys)) + for i, key := range keys { + ids[i] = primitives.AuthorityIDWeight{ + AuthorityID: key.Public(), + AuthorityWeight: 1, + } + } + return ids +} +func TestVoter(t *testing.T) { + // async fn finalize_3_voters_no_observers() { + t.Run("finalize_3_voters_no_observers", func(t *testing.T) { + // sp_tracing::try_init_simple(); + // let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; + peers := []ed25519.Keyring{ed25519.Alice, ed25519.Bob, ed25519.Charlie} + // let voters = make_ids(peers); + voters := makeIDs(peers) + _ = voters + + // let mut net = GrandpaTestNet::new(TestApi::new(voters), 3, 0); + // tokio::spawn(initialize_grandpa(&mut net, peers)); + // net.peer(0).push_blocks(20, false); + // net.run_until_sync().await; + // let hashof20 = net.peer(0).client().info().best_hash; + + // for i in 0..3 { + // assert_eq!(net.peer(i).client().info().best_number, 20, "Peer #{} failed to sync", i); + // assert_eq!(net.peer(i).client().info().best_hash, hashof20, "Peer #{} failed to sync", i); + // } + + // let net = Arc::new(Mutex::new(net)); + // run_to_completion(20, net.clone(), peers).await; + + // // all peers should have stored the justification for the best finalized block #20 + // for peer_id in 0..3 { + // let client = net.lock().peers[peer_id].client().as_client(); + // let justification = + // crate::aux_schema::best_justification::<_, Block>(&*client).unwrap().unwrap(); + + // assert_eq!(justification.justification.commit.target_number, 20); + // } + }) +} diff --git a/internal/client/consensus/grandpa/import.go b/internal/client/consensus/grandpa/import.go new file mode 100644 index 0000000000..faf3edb1ed --- /dev/null +++ b/internal/client/consensus/grandpa/import.go @@ -0,0 +1,1383 @@ +package grandpa + +import ( + "fmt" + "sync" + + "github.com/ChainSafe/gossamer/internal/client/api" + "github.com/ChainSafe/gossamer/internal/client/api/utils" + client_common "github.com/ChainSafe/gossamer/internal/client/consensus/common" + shareddata "github.com/ChainSafe/gossamer/internal/client/consensus/common/shared-data" + "github.com/ChainSafe/gossamer/internal/primitives/blockchain" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" + "github.com/ChainSafe/gossamer/internal/primitives/crypto/hashing" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/storage" + forktree "github.com/ChainSafe/gossamer/internal/utils/fork-tree" + "github.com/ChainSafe/gossamer/pkg/scale" +) + +// enum AppliedChanges { +type importAppliedChanges interface { + needsJustification() bool +} + +// Standard(bool), // true if the change is ready to be applied (i.e. it's a root) +type importAppliedChangesStandard bool + +// Forced(NewAuthoritySet), +type importAppliedChangesForced[H runtime.Hash, N runtime.Number] newAuthoritySet[H, N] + +// None, +type importAppliedChangesNone struct{} + +func (importAppliedChangesStandard) needsJustification() bool { + return true +} +func (importAppliedChangesForced[H, N]) needsJustification() bool { + return false +} +func (importAppliedChangesNone) needsJustification() bool { + return false +} + +// } + +type justInCase[H runtime.Hash, N runtime.Number] struct { + old AuthoritySet[H, N] + shareddata.SharedDataLocked[AuthoritySet[H, N]] +} + +// struct PendingSetChanges { +type pendingSetChanges[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], +] struct { + // just_in_case: Option<( + // + // AuthoritySet>, + // SharedDataLockedUpgradable>>, + // + // )>, + justInCase *justInCase[H, N] + // applied_changes: AppliedChanges>, + appliedChanges importAppliedChanges + // do_pause: bool, + doPause bool +} + +// revert the pending set change explicitly. +// fn revert(self) {} +func (pendingSetChanges[H, N, Hasher, Header]) revert() {} + +// fn defuse(mut self) -> (AppliedChanges>, bool) { +// self.just_in_case = None; +// let applied_changes = std::mem::replace(&mut self.applied_changes, AppliedChanges::None); +// (applied_changes, self.do_pause) +// } +func (psc *pendingSetChanges[H, N, Hasher, Header]) defuse() (importAppliedChanges, bool) { + psc.justInCase = nil + appliedChanges := psc.appliedChanges + psc.appliedChanges = importAppliedChangesNone{} + return appliedChanges, psc.doPause +} + +// impl Drop for PendingSetChanges { +// fn drop(&mut self) { +// if let Some((old_set, mut authorities)) = self.just_in_case.take() { +// *authorities.upgrade() = old_set; +// } +// } +// } +func (psc *pendingSetChanges[H, N, Hasher, Header]) drop() { + // if let Some((oldSet, mut authorities)) = self.justInCase.take() { + // *authorities.upgrade() = oldSet; + // } + if psc.justInCase != nil { + jic := psc.justInCase + psc.justInCase = nil + oldSet := jic.old + locked := jic.SharedDataLocked + *locked.MutRef() = oldSet + defer locked.Unlock() + } +} + +// / A block-import handler for GRANDPA. +// / +// / This scans each imported block for signals of changing authority set. +// / If the block being imported enacts an authority set change then: +// / - If the current authority set is still live: we import the block +// / - Otherwise, the block must include a valid justification. +// / +// / When using GRANDPA, the block import worker should be using this block import +// / object.im +// pub struct GrandpaBlockImport { +type GrandpaBlockImport[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +] struct { + // inner: Arc, + inner ClientForGrandpa[H, N, Hasher, Header, E] + // justification_import_period: u32, + justificationImportPeriod uint32 + // select_chain: SC, + selectChain common.SelectChain[H, N, Header] + // authority_set: SharedAuthoritySet>, + authoritySet *SharedAuthoritySet[H, N] + // send_voter_commands: TracingUnboundedSender>>, + sendVoterCommands chan voterCommand + // authority_set_hard_forks: + // + // Mutex>>>, + authoritySetHardForks map[H]PendingChange[H, N] + authoritySetHardForksMtx sync.Mutex + // + // justification_sender: GrandpaJustificationSender, + justificationSender GrandpaJustificationSender[H, N, Header] + // telemetry: Option, + // TODO: telemetry + // _phantom: PhantomData, +} + +// impl GrandpaBlockImport { +// pub(crate) fn new( +// inner: Arc, +// justification_import_period: u32, +// select_chain: SC, +// authority_set: SharedAuthoritySet>, +// send_voter_commands: TracingUnboundedSender>>, +// authority_set_hard_forks: Vec<(SetId, PendingChange>)>, +// justification_sender: GrandpaJustificationSender, +// telemetry: Option, +// ) -> GrandpaBlockImport { +func newGrandpaBlockImport[ + H runtime.Hash, + N runtime.Number, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], + E runtime.Extrinsic, +]( + inner ClientForGrandpa[H, N, Hasher, Header, E], + justificationImportPeriod uint32, + selectChain common.SelectChain[H, N, Header], + sharedAuthoritySet *SharedAuthoritySet[H, N], + sendVoterCommands chan voterCommand, + authoritySetHardForks []struct { + SetID + PendingChange[H, N] + }, + justificationSender GrandpaJustificationSender[H, N, Header], + // TODO: telemetry +) *GrandpaBlockImport[H, N, Hasher, Header, E] { + // check for and apply any forced authority set hard fork that applies + // to the *current* authority set. + // if let Some((_, change)) = authority_set_hard_forks + // .iter() + // .find(|(set_id, _)| *set_id == authority_set.set_id()) + // { + // authority_set.inner().current_authorities = change.next_authorities.clone(); + // } + for _, hardFork := range authoritySetHardForks { + setID := hardFork.SetID + if setID == SetID(sharedAuthoritySet.SetID()) { + authoritySet, unlock := sharedAuthoritySet.inner.DataMut() + // authoritySet.mtx.Lock() + authoritySet.CurrentAuthorities = hardFork.PendingChange.NextAuthorities + // authoritySet.mtx.Unlock() + unlock() + } + } + + // index authority set hard forks by block hash so that they can be used + // by any node syncing the chain and importing a block hard fork + // authority set changes. + // let authority_set_hard_forks = authority_set_hard_forks + // .into_iter() + // .map(|(_, change)| (change.canon_hash, change)) + // .collect::>(); + var authoritySetHardForksMap = make(map[H]PendingChange[H, N]) + for _, hardFork := range authoritySetHardForks { + authoritySetHardForksMap[hardFork.PendingChange.CanonHash] = hardFork.PendingChange + } + + // check for and apply any forced authority set hard fork that apply to + // any *pending* standard changes, checking by the block hash at which + // they were announced. + // { + // let mut authority_set = authority_set.inner(); + + // authority_set.pending_standard_changes = + // authority_set.pending_standard_changes.clone().map(&mut |hash, _, original| { + // authority_set_hard_forks.get(hash).cloned().unwrap_or(original) + // }); + // } + // authoritySet.mtx.Lock() + // authSet := &authoritySet.inner + authoritySet, unlock := sharedAuthoritySet.inner.DataMut() + + authoritySet.PendingStandardChanges = forktree.Map(authoritySet.PendingStandardChanges, func(hash H, _ N, original PendingChange[H, N]) PendingChange[H, N] { + if change, ok := authoritySetHardForksMap[hash]; ok { + return change + } + return original + }) + // authoritySet.mtx.Unlock() + unlock() + + // GrandpaBlockImport { + // inner, + // justification_import_period, + // select_chain, + // authority_set, + // send_voter_commands, + // authority_set_hard_forks: Mutex::new(authority_set_hard_forks), + // justification_sender, + // telemetry, + // _phantom: PhantomData, + // } + return &GrandpaBlockImport[H, N, Hasher, Header, E]{ + inner: inner, + justificationImportPeriod: justificationImportPeriod, + selectChain: selectChain, + authoritySet: sharedAuthoritySet, + sendVoterCommands: sendVoterCommands, + authoritySetHardForks: authoritySetHardForksMap, + justificationSender: justificationSender, + } +} + +// / Checks the given header for a consensus digest signalling a **standard** scheduled change and +// / extracts it. +// pub fn find_scheduled_change( +// +// header: &B::Header, +// +// ) -> Option>> { +func FindScheduledChange[ + H runtime.Hash, + N runtime.Number, + Header runtime.Header[N, H], +]( + header Header, +) *grandpa.ScheduledChange[N] { + // let id = OpaqueDigestItemId::Consensus(&GRANDPA_ENGINE_ID); + id := runtime.OpaqueDigestItemIDConsensus(grandpa.GrandpaEngineID) + + // let filter_log = |log: ConsensusLog>| match log { + // ConsensusLog::ScheduledChange(change) => Some(change), + // _ => None, + // }; + filterLog := func(log grandpa.ConsensusLog) *grandpa.ScheduledChange[N] { + scheduledChange, ok := log.(grandpa.ConensusLogScheduledChange[N]) + if !ok { + return nil + } + orig := grandpa.ScheduledChange[N](scheduledChange) + return &orig + } + // find the first consensus digest with the right ID which converts to + // the right kind of consensus log. + // header.digest().convert_first(|l| l.try_to(id).and_then(filter_log)) + for _, log := range header.Digest().Logs { + logVDT := runtime.DigestItemTryTo[grandpa.ConsensusLogVDT[N]](log, id) + if logVDT == nil { + continue + } + val, err := logVDT.Value() + if err != nil { + continue + } + log := val.(grandpa.ConsensusLog) + sc := filterLog(log) + if sc != nil { + return sc + } + } + return nil +} + +type ForcedChange[H runtime.Hash, N runtime.Number] struct { + MedianLastFinalized N + grandpa.ScheduledChange[N] +} + +// / Checks the given header for a consensus digest signalling a **forced** scheduled change and +// / extracts it. +// pub fn find_forced_change( +// +// header: &B::Header, +// +// ) -> Option<(NumberFor, ScheduledChange>)> { +func FindForcedChange[ + H runtime.Hash, + N runtime.Number, + Header runtime.Header[N, H], +]( + header Header, +) *ForcedChange[H, N] { + // let id = OpaqueDigestItemId::Consensus(&GRANDPA_ENGINE_ID); + id := runtime.OpaqueDigestItemIDConsensus(grandpa.GrandpaEngineID) + + // let filter_log = |log: ConsensusLog>| match log { + // ConsensusLog::ForcedChange(delay, change) => Some((delay, change)), + // _ => None, + // }; + filterLog := func(log grandpa.ConsensusLog) *ForcedChange[H, N] { + forcedChange, ok := log.(grandpa.ConsensusLogForcedChange[N]) + if !ok { + return nil + } + return &ForcedChange[H, N]{ + MedianLastFinalized: forcedChange.Delay, + ScheduledChange: forcedChange.ScheduledChange, + } + } + + // // find the first consensus digest with the right ID which converts to + // // the right kind of consensus log. + // header.digest().convert_first(|l| l.try_to(id).and_then(filter_log)) + for _, log := range header.Digest().Logs { + logVDT := runtime.DigestItemTryTo[grandpa.ConsensusLogVDT[N]](log, id) + if logVDT == nil { + continue + } + val, err := logVDT.Value() + if err != nil { + continue + } + log := val.(grandpa.ConsensusLog) + fc := filterLog(log) + if fc != nil { + return fc + } + } + return nil +} + +// impl GrandpaBlockImport +// where +// +// NumberFor: finality_grandpa::BlockNumberOps, +// BE: Backend, +// Client: ClientForGrandpa, +// Client::Api: GrandpaApi, +// for<'a> &'a Client: BlockImport, +// +// { +// // check for a new authority set change. +// fn check_new_change( +// &self, +// header: &Block::Header, +// hash: Block::Hash, +// ) -> Option>> { +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) checkNewChange( + header Header, + hash H, +) *PendingChange[H, N] { + // check for forced authority set hard forks + // if let Some(change) = self.authority_set_hard_forks.lock().get(&hash) { + // return Some(change.clone()) + // } + gbi.authoritySetHardForksMtx.Lock() + change, ok := gbi.authoritySetHardForks[hash] + gbi.authoritySetHardForksMtx.Unlock() + if ok { + return &change + } + + // check for forced change. + // if let Some((median_last_finalized, change)) = find_forced_change::(header) { + // return Some(PendingChange { + // next_authorities: change.next_authorities, + // delay: change.delay, + // canon_height: *header.number(), + // canon_hash: hash, + // delay_kind: DelayKind::Best { median_last_finalized }, + // }) + // } + fc := FindForcedChange[H, N](header) + if fc != nil { + return &PendingChange[H, N]{ + NextAuthorities: fc.ScheduledChange.NextAuthorities, + Delay: fc.ScheduledChange.Delay, + CanonHeight: header.Number(), + CanonHash: hash, + DelayKind: delayKindBest[N]{MedianLastFinalized: fc.MedianLastFinalized}, + } + } + + // check normal scheduled change. + // let change = find_scheduled_change::(header)?; + // Some(PendingChange { + // next_authorities: change.next_authorities, + // delay: change.delay, + // canon_height: *header.number(), + // canon_hash: hash, + // delay_kind: DelayKind::Finalized, + // }) + scheduled := FindScheduledChange[H, N](header) + return &PendingChange[H, N]{ + NextAuthorities: scheduled.NextAuthorities, + Delay: scheduled.Delay, + CanonHeight: header.Number(), + CanonHash: hash, + DelayKind: delayKindFinalized{}, + } +} + +// struct InnerGuard<'a, H, N> { +// old: Option>, +// guard: Option>>, +// } +type innerGuard[H runtime.Hash, N runtime.Number] struct { + old *AuthoritySet[H, N] + guard *shareddata.SharedDataLocked[AuthoritySet[H, N]] +} + +// impl<'a, H, N> InnerGuard<'a, H, N> { +// fn as_mut(&mut self) -> &mut AuthoritySet { +// self.guard.as_mut().expect("only taken on deconstruction; qed") +// } +func (ig *innerGuard[H, N]) asMut() *AuthoritySet[H, N] { + if ig.guard == nil { + panic("guard is nil; only taken on deconstruction; qed") + } + return ig.guard.MutRef() +} + +// fn set_old(&mut self, old: AuthoritySet) { +// if self.old.is_none() { +// // ignore "newer" old changes. +// self.old = Some(old); +// } +// } +func (ig *innerGuard[H, N]) setOld(old AuthoritySet[H, N]) { + if ig.old == nil { + // ignore "newer" old changes. + ig.old = &old + } +} + +// fn consume( +// mut self, +// ) -> Option<(AuthoritySet, SharedDataLocked<'a, AuthoritySet>)> { +// self.old +// .take() +// .map(|old| (old, self.guard.take().expect("only taken on deconstruction; qed"))) +// } +// } +type consumed[H runtime.Hash, N runtime.Number] struct { + old AuthoritySet[H, N] + shareddata.SharedDataLocked[AuthoritySet[H, N]] +} + +func (ig *innerGuard[H, N]) consume() *consumed[H, N] { + old := ig.old + ig.old = nil + if old == nil { + return nil + } + if ig.guard == nil { + panic("guard is nil; only taken on deconstruction; qed") + } + return &consumed[H, N]{ + old: *old, + SharedDataLocked: *ig.guard, + } +} + +// impl<'a, H, N> Drop for InnerGuard<'a, H, N> { +// fn drop(&mut self) { +// if let (Some(mut guard), Some(old)) = (self.guard.take(), self.old.take()) { +// *guard = old; +// } +// } +// } +func (ig *innerGuard[H, N]) drop() { + // if let (Some(mut guard), Some(old)) = (self.guard.take(), self.old.take()) { + // *guard = old; + // } + guard := ig.guard + ig.guard = nil + old := ig.old + ig.old = nil + if guard != nil && old != nil { + *ig.guard.MutRef() = *ig.old + } +} + +// fn make_authorities_changes( +// +// &self, +// block: &mut BlockImportParams, +// hash: Block::Hash, +// initial_sync: bool, +// +// ) -> Result, ConsensusError> { +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) makeAuthoritiesChanges( + block *client_common.BlockImportParams[H, N, E, Header], + hash H, + initialSync bool, +) (*pendingSetChanges[H, N, Hasher, Header], error) { + // when we update the authorities, we need to hold the lock + // until the block is written to prevent a race if we need to restore + // the old authority set on error or panic. + + // let number = *(block.header.number()); + // let maybe_change = self.check_new_change(&block.header, hash); + number := block.Header.Number() + maybeChange := gbi.checkNewChange(block.Header, hash) + + // returns a function for checking whether a block is a descendent of another + // consistent with querying client directly after importing the block. + // let parent_hash = *block.header.parent_hash(); + // let is_descendent_of = is_descendent_of(&*self.inner, Some((hash, parent_hash))); + parentHash := block.Header.ParentHash() + isDescendentOf := utils.IsDescendantOf(gbi.inner, &utils.HashParent[H]{Hash: hash, Parent: parentHash}) + // let mut guard = InnerGuard { guard: Some(self.authority_set.inner_locked()), old: None }; + locked := gbi.authoritySet.inner.Locked() + defer locked.Unlock() + guard := innerGuard[H, N]{ + old: nil, + guard: &locked, + } + + // whether to pause the old authority set -- happens after import + // of a forced change block. + // let mut do_pause = false; + var doPause bool + + // add any pending changes. + // if let Some(change) = maybe_change { + // let old = guard.as_mut().clone(); + // guard.set_old(old); + + // if let DelayKind::Best { .. } = change.delay_kind { + // do_pause = true; + // } + + // guard + // .as_mut() + // .add_pending_change(change, &is_descendent_of) + // .map_err(|e| ConsensusError::ClientImport(e.to_string()))?; + // } + if maybeChange != nil { + change := *maybeChange + old := guard.asMut().Clone() + guard.setOld(old) + + if _, ok := change.DelayKind.(delayKindBest[N]); ok { + doPause = true + } + + err := guard.asMut().addPendingChange(change, isDescendentOf) + if err != nil { + return nil, err + } + } + + // let applied_changes = { + // let forced_change_set = guard + // .as_mut() + // .apply_forced_changes( + // hash, + // number, + // &is_descendent_of, + // initial_sync, + // self.telemetry.clone(), + // ) + // .map_err(|e| ConsensusError::ClientImport(e.to_string())) + // .map_err(ConsensusError::from)?; + var appliedChanges importAppliedChanges + forcedChangeSet, err := guard.asMut().applyForcedChanges( + hash, + number, + isDescendentOf, + ) + if err != nil { + return nil, err + } + + // if let Some((median_last_finalized_number, new_set)) = forced_change_set { + if forcedChangeSet != nil { + medianLastFinalizedNumber := forcedChangeSet.median + newSet := forcedChangeSet.set + // let new_authorities = { + // let (set_id, new_authorities) = new_set.current(); + setID, newAuthorities := newSet.current() + + // we will use the median last finalized number as a hint + // for the canon block the new authority set should start + // with. we use the minimum between the median and the local + // best finalized block. + // let best_finalized_number = self.inner.info().finalized_number; + // let canon_number = best_finalized_number.min(median_last_finalized_number); + // let canon_hash = self.inner.hash(canon_number) + // .map_err(|e| ConsensusError::ClientImport(e.to_string()))? + // .expect( + // "the given block number is less or equal than the current best finalized number; \ + // current best finalized number must exist in chain; qed." + // ); + bestFinalizedNumber := gbi.inner.Info().FinalizedNumber + canonNumber := bestFinalizedNumber + if medianLastFinalizedNumber < bestFinalizedNumber { + canonNumber = medianLastFinalizedNumber + } + canonHash, err := gbi.inner.Hash(canonNumber) + if err != nil { + return nil, err + } + if canonHash == nil { + panic("the given block number is less or equal than the current best finalized number; current best finalized number must exist in chain; qed.") + } + + // NewAuthoritySet { + // canon_number, + // canon_hash, + // set_id, + // authorities: new_authorities.to_vec(), + // } + // }; + // let old = ::std::mem::replace(guard.as_mut(), new_set); + // guard.set_old(old); + newAuthoritySet := newAuthoritySet[H, N]{ + CanonNumber: canonNumber, + CanonHash: *canonHash, + SetID: grandpa.SetID(setID), + Authorities: newAuthorities, + } + old := guard.asMut().Clone() + guard.setOld(old) + *guard.asMut() = newSet + + // AppliedChanges::Forced(new_authorities) + appliedChanges = importAppliedChangesForced[H, N](newAuthoritySet) + } else { + // let did_standard = guard + // .as_mut() + // .enacts_standard_change(hash, number, &is_descendent_of) + // .map_err(|e| ConsensusError::ClientImport(e.to_string())) + // .map_err(ConsensusError::from)?; + + // if let Some(root) = did_standard { + // AppliedChanges::Standard(root) + // } else { + // AppliedChanges::None + // } + // } + didStandard, err := guard.asMut().EnactsStandardChange(hash, number, isDescendentOf) + if err != nil { + return nil, err + } + + if didStandard != nil { + appliedChanges = importAppliedChangesStandard(*didStandard) + } else { + appliedChanges = importAppliedChangesNone{} + } + } + + // consume the guard safely and write necessary changes. + // let just_in_case = guard.consume(); + justInCaseConsumed := guard.consume() + // if let Some((_, ref authorities)) = just_in_case { + if justInCaseConsumed != nil { + authorities := &justInCaseConsumed.SharedDataLocked + // let authorities_change = match applied_changes { + // AppliedChanges::Forced(ref new) => Some(new), + // AppliedChanges::Standard(_) => None, // the change isn't actually applied yet. + // AppliedChanges::None => None, + // }; + var authoritiesChange *newAuthoritySet[H, N] + switch appliedChanges := appliedChanges.(type) { + case importAppliedChangesForced[H, N]: + newSet := newAuthoritySet[H, N](appliedChanges) + authoritiesChange = &newSet + case importAppliedChangesStandard: + // the change isn't actually applied yet. + case importAppliedChangesNone: + // no change + default: + panic("unreachable") + } + // crate::aux_schema::update_authority_set::( + // authorities, + // authorities_change, + // |insert| { + // block + // .auxiliary + // .extend(insert.iter().map(|(k, v)| (k.to_vec(), Some(v.to_vec())))) + // }, + // ); + updateAuthoritySet(authorities.Data(), authoritiesChange, func(insertions []api.KeyValue) error { + converted := make([]api.AuxDataOperation, len(insertions)) + for i, kv := range insertions { + converted[i] = api.AuxDataOperation{ + Key: kv.Key, + Data: kv.Value, + } + } + block.Auxiliary = append(block.Auxiliary, converted...) + return nil + }) + } + + // let just_in_case = just_in_case.map(|(o, i)| (o, i.release_mutex())); + var jic *justInCase[H, N] + if justInCaseConsumed != nil { + jic = &justInCase[H, N]{ + old: justInCaseConsumed.old, + SharedDataLocked: justInCaseConsumed.SharedDataLocked, + } + } + + // Ok(PendingSetChanges { just_in_case, applied_changes, do_pause }) + return &pendingSetChanges[H, N, Hasher, Header]{ + justInCase: jic, + appliedChanges: appliedChanges, + doPause: doPause, + }, nil +} + +// /// Read current set id form a given state. +// fn current_set_id(&self, hash: Block::Hash) -> Result { +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) currentSetID(hash H) (grandpa.SetID, error) { + // let runtime_version = self.inner.runtime_api().version(hash).map_err(|e| { + // ConsensusError::ClientImport(format!( + // "Unable to retrieve current runtime version. {}", + // e + // )) + // })?; + runtimeVersion, err := gbi.inner.RuntimeAPI().Version(hash) + if err != nil { + return 0, err + } + + var GrandpaID = [8]byte{237, 153, 197, 172, 178, 94, 237, 245} + apiVersion := runtimeVersion.APIVersion(GrandpaID) + + // if runtime_version + // .api_version(&>::ID) + // .map_or(false, |v| v < 3) + // { + if apiVersion != nil && *apiVersion < 3 { + // The new API is not supported in this runtime. Try reading directly from storage. + // This code may be removed once warp sync to an old runtime is no longer needed. + // for prefix in ["GrandpaFinality", "Grandpa"] { + for _, prefix := range []string{"GrandpaFinality", "Grandpa"} { + // let k = [ + // sp_crypto_hashing::twox_128(prefix.as_bytes()), + // sp_crypto_hashing::twox_128(b"CurrentSetId"), + // ] + // .concat(); + k0 := hashing.Twox128([]byte(prefix)) + k1 := hashing.Twox128([]byte("CurrentSetId")) + k := k0[:] + k = append(k, k1[:]...) + // if let Ok(Some(id)) = + // self.inner.storage(hash, &sc_client_api::StorageKey(k.to_vec())) + // { + // if let Ok(id) = SetId::decode(&mut id.0.as_ref()) { + // return Ok(id) + // } + // } + id, _ := gbi.inner.Storage(hash, storage.StorageKey(k)) + if id != nil { + var setID grandpa.SetID + err := scale.Unmarshal(*id, &setID) + if err == nil { + return setID, nil + } + } + } + // Err(ConsensusError::ClientImport("Unable to retrieve current set id.".into())) + return 0, fmt.Errorf("unable to retrieve current set id") + } else { + // self.inner + // .runtime_api() + // .current_set_id(hash) + // .map_err(|e| ConsensusError::ClientImport(e.to_string())) + setID, err := gbi.inner.RuntimeAPI().CurrentSetID(hash) + if err != nil { + return 0, err + } + return setID, nil + } +} + +// /// Import whole new state and reset authority set. +// async fn import_state( +// +// &self, +// mut block: BlockImportParams, +// +// ) -> Result { +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) importState( + block *client_common.BlockImportParams[H, N, E, Header], +) (client_common.ImportResult, error) { + // let hash = block.post_hash(); + // let number = *block.header.number(); + hash := block.GetPostHash() + number := block.Header.Number() + + // Force imported state finality. + // block.finalized = true; + // let import_result = (&*self.inner).import_block(block).await; + block.Finalized = true + importResult, err := gbi.inner.ImportBlock(block) + // match import_result { + if err == nil { + // Ok(ImportResult::Imported(aux)) => { + switch importResult := importResult.(type) { + case client_common.ImportResultImported: + aux := client_common.ImportedAux(importResult) + // We've just imported a new state. We trust the sync module has verified + // finality proofs and that the state is correct and final. + // So we can read the authority list and set id from the state. + // self.authority_set_hard_forks.lock().clear(); + gbi.authoritySetHardForksMtx.Lock() + gbi.authoritySetHardForks = make(map[H]PendingChange[H, N]) + gbi.authoritySetHardForksMtx.Unlock() + // let authorities = self + // .inner + // .runtime_api() + // .grandpa_authorities(hash) + // .map_err(|e| ConsensusError::ClientImport(e.to_string()))?; + authorities, err := gbi.inner.RuntimeAPI().GrandpaAuthorities(hash) + if err != nil { + return nil, err + } + // let set_id = self.current_set_id(hash)?; + setID, err := gbi.currentSetID(hash) + if err != nil { + return nil, err + } + // let authority_set = AuthoritySet::new( + // authorities.clone(), + // set_id, + // fork_tree::ForkTree::new(), + // Vec::new(), + // AuthoritySetChanges::empty(), + // ) + // .ok_or_else(|| ConsensusError::ClientImport("Invalid authority list".into()))?; + authoritySet, err := NewAuthoritySet[H, N]( + authorities, + uint64(setID), + forktree.NewForkTree[H, N, PendingChange[H, N]](), + []PendingChange[H, N]{}, + AuthoritySetChanges[N]{}, + ) + if err != nil { + return nil, err + } + + // *self.authority_set.inner_locked() = authority_set.clone(); + locked := gbi.authoritySet.inner.Locked() + *locked.MutRef() = authoritySet.Clone() + defer locked.Unlock() + + // crate::aux_schema::update_authority_set::( + // &authority_set, + // None, + // |insert| self.inner.insert_aux(insert, []), + // ) + // .map_err(|e| ConsensusError::ClientImport(e.to_string()))?; + err = updateAuthoritySet( + locked.Data(), + nil, + func(insertions []api.KeyValue) error { + return gbi.inner.InsertAux(insertions, nil) + }, + ) + if err != nil { + return nil, err + } + // let new_set = + // NewAuthoritySet { canon_number: number, canon_hash: hash, set_id, authorities }; + // let _ = self + // .send_voter_commands + // .unbounded_send(VoterCommand::ChangeAuthorities(new_set)); + // Ok(ImportResult::Imported(aux)) + newSet := newAuthoritySet[H, N]{ + CanonNumber: number, + CanonHash: hash, + SetID: grandpa.SetID(setID), + Authorities: authorities, + } + gbi.sendVoterCommands <- voterCommandChangeAuthorities[H, N](newSet) + return client_common.ImportResultImported(aux), nil + // }, + case client_common.ImportResultAlreadyInChain, + client_common.ImportResultKnownBad, + client_common.ImportResultMissingState, + client_common.ImportResultUnknownParent: + // Ok(r) => Ok(r), + return importResult, nil + default: + panic("unreachable") + } + + } else { + return nil, err + } + // Err(e) => Err(ConsensusError::ClientImport(e.to_string())), + // } + // } +} + +// impl BlockImport for GrandpaBlockImport +// where +// NumberFor: finality_grandpa::BlockNumberOps, +// BE: Backend, +// Client: ClientForGrandpa, +// Client::Api: GrandpaApi, +// for<'a> &'a Client: BlockImport, +// SC: Send + Sync, +// { +// type Error = ConsensusError; + +// async fn import_block( +// +// &self, +// mut block: BlockImportParams, +// +// ) -> Result { +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) importBlock( + block *client_common.BlockImportParams[H, N, E, Header], +) (client_common.ImportResult, error) { + // let hash = block.post_hash(); + // let number = *block.header.number(); + hash := block.GetPostHash() + number := block.Header.Number() + + // early exit if block already in chain, otherwise the check for + // authority changes will error when trying to re-import a change block + // match self.inner.status(hash) { + // Ok(BlockStatus::InChain) => { + // // Strip justifications when re-importing an existing block. + // let _justifications = block.justifications.take(); + // return (&*self.inner).import_block(block).await + // }, + // Ok(BlockStatus::Unknown) => {}, + // Err(e) => return Err(ConsensusError::ClientImport(e.to_string())), + // } + status, err := gbi.inner.Status(hash) + if err != nil { + return nil, err + } + if status == blockchain.BlockStatusInChain { + // Strip justifications when re-importing an existing block. + block.Justifications = nil + return gbi.inner.ImportBlock(block) + } + + // if block.with_state() { + // return self.import_state(block).await + // } + if block.WithState() { + return gbi.importState(block) + } + + // if number <= self.inner.info().finalized_number { + if number <= gbi.inner.Info().FinalizedNumber { + // Importing an old block. Just save justifications and authority set changes + // if self.check_new_change(&block.header, hash).is_some() { + // if block.justifications.is_none() { + // return Err(ConsensusError::ClientImport( + // "Justification required when importing \ + // an old block with authority set change." + // .into(), + // )) + // } + // let mut authority_set = self.authority_set.inner_locked(); + // authority_set.authority_set_changes.insert(number); + // crate::aux_schema::update_authority_set::( + // &authority_set, + // None, + // |insert| { + // block + // .auxiliary + // .extend(insert.iter().map(|(k, v)| (k.to_vec(), Some(v.to_vec())))) + // }, + // ); + // } + if gbi.checkNewChange(block.Header, hash) != nil { + if block.Justifications == nil { + return nil, fmt.Errorf("justification required when importing an old block with authority set change") + } + locked := gbi.authoritySet.inner.Locked() + authoritySet := locked.MutRef() + authoritySet.AuthoritySetChanges.insert(number) + err := updateAuthoritySet( + *authoritySet, + nil, + func(insertions []api.KeyValue) error { + converted := make([]api.AuxDataOperation, len(insertions)) + for i, kv := range insertions { + converted[i] = api.AuxDataOperation{ + Key: kv.Key, + Data: kv.Value, + } + } + block.Auxiliary = append(block.Auxiliary, converted...) + return nil + }, + ) + if err != nil { + return nil, err + } + } + // return (&*self.inner).import_block(block).await + return gbi.inner.ImportBlock(block) + } + + // on initial sync we will restrict logging under info to avoid spam. + // let initial_sync = block.origin == BlockOrigin::NetworkInitialSync; + initialSync := block.Origin == common.NetworkInitialSyncBlockOrigin + + // let pending_changes = self.make_authorities_changes(&mut block, hash, initial_sync)?; + pendingChanges, err := gbi.makeAuthoritiesChanges(block, hash, initialSync) + if err != nil { + return nil, err + } + defer pendingChanges.drop() + + // we don't want to finalize on `inner.import_block` + // let mut justifications = block.justifications.take(); + // let import_result = (&*self.inner).import_block(block).await; + justifications := block.Justifications + block.Justifications = nil + importResult, err := gbi.inner.ImportBlock(block) + + // let mut imported_aux = { + // match import_result { + // Ok(ImportResult::Imported(aux)) => aux, + // Ok(r) => { + // debug!( + // target: LOG_TARGET, + // "Restoring old authority set after block import result: {:?}", r, + // ); + // pending_changes.revert(); + // return Ok(r) + // }, + // Err(e) => { + // debug!( + // target: LOG_TARGET, + // "Restoring old authority set after block import error: {}", e, + // ); + // pending_changes.revert(); + // return Err(ConsensusError::ClientImport(e.to_string())) + // }, + // } + // }; + if err != nil { + logger.Debugf("Restoring old authority set after block import error: %s", err) + pendingChanges.revert() + return nil, err + } + var importedAux client_common.ImportedAux + switch importResult := importResult.(type) { + case client_common.ImportResultImported: + importedAux = client_common.ImportedAux(importResult) + default: + logger.Debugf("Restoring old authority set after block import result: %v", importResult) + pendingChanges.revert() + return importResult, nil + } + + // let (applied_changes, do_pause) = pending_changes.defuse(); + appliedChanges, doPause := pendingChanges.defuse() + + // Send the pause signal after import but BEFORE sending a `ChangeAuthorities` message. + // if do_pause { + // let _ = self.send_voter_commands.unbounded_send(VoterCommand::Pause( + // "Forced change scheduled after inactivity".to_string(), + // )); + // } + if doPause { + gbi.sendVoterCommands <- voterCommandPause("Forced change scheduled after inactivity") + } + + // let needs_justification = applied_changes.needs_justification(); + needsJustification := appliedChanges.needsJustification() + + // match applied_changes { + switch appliedChanges := appliedChanges.(type) { + // AppliedChanges::Forced(new) => { + case importAppliedChangesForced[H, N]: + // NOTE: when we do a force change we are "discrediting" the old set so we + // ignore any justifications from them. this block may contain a justification + // which should be checked and imported below against the new authority + // triggered by this forced change. the new grandpa voter will start at the + // last median finalized block (which is before the block that enacts the + // change), full nodes syncing the chain will not be able to successfully + // import justifications for those blocks since their local authority set view + // is still of the set before the forced change was enacted, still after #1867 + // they should import the block and discard the justification, and they will + // then request a justification from sync if it's necessary (which they should + // then be able to successfully validate). + // let _ = + // self.send_voter_commands.unbounded_send(VoterCommand::ChangeAuthorities(new)); + gbi.sendVoterCommands <- voterCommandChangeAuthorities[H, N](newAuthoritySet[H, N](appliedChanges)) + // we must clear all pending justifications requests, presumably they won't be + // finalized hence why this forced changes was triggered + // imported_aux.clear_justification_requests = true; + importedAux.ClearJustificationRequests = true + + // AppliedChanges::Standard(false) => { + case importAppliedChangesStandard: + // this is a standard change, we don't apply it yet, but we will send a + // we can't apply this change yet since there are other dependent changes that we + // need to apply first, drop any justification that might have been provided with + // the block to make sure we request them from `sync` which will ensure they'll be + // applied in-order. + // justifications.take(); + justifications = nil + default: + } + // } + + // let grandpa_justification = + // justifications.and_then(|just| just.into_justification(GRANDPA_ENGINE_ID)); + var grandpaJustification *runtime.EncodedJustification + if justifications != nil { + grandpaJustification = justifications.IntoJustification(grandpa.GrandpaEngineID) + } + + // match grandpa_justification { + // Some(justification) => { + if grandpaJustification != nil { + // if environment::should_process_justification( + // &*self.inner, + // self.justification_import_period, + // number, + // needs_justification, + // ) { + if shouldProcessJustification( + gbi.inner, + gbi.justificationImportPeriod, + number, + needsJustification, + ) { + // let import_res = self.import_justification( + // hash, + // number, + // (GRANDPA_ENGINE_ID, justification), + // needs_justification, + // initial_sync, + // ); + err := gbi.importJustification( + hash, + number, + runtime.Justification{ + ConsensusEngineID: grandpa.GrandpaEngineID, + EncodedJustification: *grandpaJustification, + }, + needsJustification, + initialSync, + ) + + // import_res.unwrap_or_else(|err| { + if err != nil { + // if needs_justification { + // debug!( + // target: LOG_TARGET, + // "Requesting justification from peers due to imported block #{} that enacts authority set change with invalid justification: {}", + // number, + // err + // ); + // imported_aux.bad_justification = true; + // imported_aux.needs_justification = true; + // } + if needsJustification { + logger.Debugf("Requesting justification from peers due to imported block #%d that enacts authority set change with invalid justification: %s", number, err) + importedAux.BadJustification = true + importedAux.NeedsJustification = true + } + } + // }); + } else { + // debug!( + // target: LOG_TARGET, + // "Ignoring unnecessary justification for block #{}", + // number, + // ); + logger.Debugf("Ignoring unnecessary justification for block #%d", number) + } + } else { + // None => + // if needs_justification { + // debug!( + // target: LOG_TARGET, + // "Imported unjustified block #{} that enacts authority set change, waiting for finality for enactment.", + // number, + // ); + + // imported_aux.needs_justification = true; + // }, + if needsJustification { + logger.Debugf("Imported unjustified block #%d that enacts authority set change, waiting for finality for enactment.", number) + importedAux.NeedsJustification = true + } + } + + // Ok(ImportResult::Imported(imported_aux)) + return client_common.ImportResultImported(importedAux), nil +} + +// async fn check_block( +// +// &self, +// block: BlockCheckParams, +// +// ) -> Result { +// self.inner.check_block(block).await +// } +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) checkBlock( + block client_common.BlockCheckParams[H, N], +) (client_common.ImportResult, error) { + return gbi.inner.CheckBlock(block) +} + +// } + +// impl GrandpaBlockImport +// where +// +// BE: Backend, +// Client: ClientForGrandpa, +// NumberFor: finality_grandpa::BlockNumberOps, +// +// { +// /// Import a block justification and finalize the block. +// /// +// /// If `enacts_change` is set to true, then finalizing this block *must* +// /// enact an authority set change, the function will panic otherwise. +// fn import_justification( +// &self, +// hash: Block::Hash, +// number: NumberFor, +// justification: Justification, +// enacts_change: bool, +// initial_sync: bool, +// ) -> Result<(), ConsensusError> { +func (gbi *GrandpaBlockImport[H, N, Hasher, Header, E]) importJustification( + hash H, + number N, + justification runtime.Justification, + enactsChange bool, + initialSync bool, +) error { + // if justification.0 != GRANDPA_ENGINE_ID { + if justification.ConsensusEngineID != grandpa.GrandpaEngineID { + // TODO: the import queue needs to be refactored to be able dispatch to the correct + // `JustificationImport` instance based on `ConsensusEngineId`, or we need to build a + // justification import pipeline similar to what we do for `BlockImport`. In the + // meantime we'll just drop the justification, since this is only used for BEEFY which + // is still WIP. + return nil + } + + // let justification = GrandpaJustification::decode_and_verify_finalizes( + // &justification.1, + // (hash, number), + // self.authority_set.set_id(), + // &self.authority_set.current_authorities(), + // ); + just, err := DecodeGrandpaJustificationVerifyFinalizes[H, N, Hasher, Header]( + justification.EncodedJustification, + HashNumber[H, N]{Hash: hash, Number: number}, + gbi.authoritySet.SetID(), + gbi.authoritySet.CurrentAuthorities(), + ) + + // let justification = match justification { + // Err(e) => return Err(ConsensusError::ClientImport(e.to_string())), + // Ok(justification) => justification, + // }; + if err != nil { + return err + } + + // let result = environment::finalize_block( + // self.inner.clone(), + // &self.authority_set, + // None, + // hash, + // number, + // justification.into(), + // initial_sync, + // Some(&self.justification_sender), + // self.telemetry.clone(), + // ); + err = finalizeBlock( + gbi.inner, + gbi.authoritySet, + nil, + hash, + number, + justificationOrCommitJustification[H, N, Header]{just}, + initialSync, + &gbi.justificationSender, + ) + // match result { + if err != nil { + // Err(CommandOrError::VoterCommand(command)) => { + // grandpa_log!( + // initial_sync, + // "👴 Imported justification for block #{} that triggers \ + // command {}, signaling voter.", + // number, + // command, + // ); + + // // send the command to the voter + // let _ = self.send_voter_commands.unbounded_send(command); + // }, + _, ok := err.(voterCommand) + if ok { + l := logger.Infof + if initialSync { + l = logger.Debugf + } + l("👴 Imported justification for block #%d that triggers command %s, signaling voter.", number, err) + + // send the command to the voter + gbi.sendVoterCommands <- err.(voterCommand) + } else { + // Err(CommandOrError::Error(e)) => + // return Err(match e { + // Error::Grandpa(error) => ConsensusError::ClientImport(error.to_string()), + // Error::Network(error) => ConsensusError::ClientImport(error), + // Error::Blockchain(error) => ConsensusError::ClientImport(error), + // Error::Client(error) => ConsensusError::ClientImport(error.to_string()), + // Error::Safety(error) => ConsensusError::ClientImport(error), + // Error::Signing(error) => ConsensusError::ClientImport(error), + // Error::Timer(error) => ConsensusError::ClientImport(error.to_string()), + // Error::RuntimeApi(error) => ConsensusError::ClientImport(error.to_string()), + // }), + return err + } + } else { + // Ok(_) => { + // assert!( + // !enacts_change, + // "returns Ok when no authority set change should be enacted; qed;" + // ); + // }, + if enactsChange { + panic("returns Ok when no authority set change should be enacted; qed;") + } + } + + // Ok(()) + // } + return nil +} diff --git a/internal/client/consensus/grandpa/justification.go b/internal/client/consensus/grandpa/justification.go index 9803bab75b..f7577e46a8 100644 --- a/internal/client/consensus/grandpa/justification.go +++ b/internal/client/consensus/grandpa/justification.go @@ -182,7 +182,7 @@ func DecodeGrandpaJustificationVerifyFinalizes[ encoded []byte, finalizedTarget HashNumber[Hash, N], setID uint64, - voters grandpa.VoterSet[string], + voters grandpa.VoterSet[primitives.AuthorityID], ) (GrandpaJustification[Hash, N, Header], error) { justification, err := DecodeJustification[Hash, N, Hasher, Header](encoded) if err != nil { @@ -203,16 +203,16 @@ func DecodeGrandpaJustificationVerifyFinalizes[ // Verify will validate the commit and the votes' ancestry proofs. func (j *GrandpaJustification[Hash, N, Header]) Verify(setID uint64, authorities primitives.AuthorityList) error { - var weights []grandpa.IDWeight[string] + var weights []grandpa.IDWeight[primitives.AuthorityID] for _, authority := range authorities { - weight := grandpa.IDWeight[string]{ - ID: string(authority.AuthorityID.Bytes()), + weight := grandpa.IDWeight[primitives.AuthorityID]{ + ID: authority.AuthorityID, Weight: uint64(authority.AuthorityWeight), } weights = append(weights, weight) } - voters := grandpa.NewVoterSet[string](weights) + voters := grandpa.NewVoterSet[primitives.AuthorityID](weights) if voters != nil { err := j.verifyWithVoterSet(setID, *voters) return err @@ -223,19 +223,19 @@ func (j *GrandpaJustification[Hash, N, Header]) Verify(setID uint64, authorities // Validate the commit and the votes' ancestry proofs. func (j *GrandpaJustification[Hash, N, Header]) verifyWithVoterSet( setID uint64, - voters grandpa.VoterSet[string], + voters grandpa.VoterSet[primitives.AuthorityID], ) error { ancestryChain := newAncestryChain[Hash, N](j.Justification.VoteAncestries) - signedPrecommits := make([]grandpa.SignedPrecommit[Hash, N, string, string], 0) + signedPrecommits := make([]grandpa.SignedPrecommit[Hash, N, string, primitives.AuthorityID], 0) for _, pc := range j.Justification.Commit.Precommits { - signedPrecommits = append(signedPrecommits, grandpa.SignedPrecommit[Hash, N, string, string]{ + signedPrecommits = append(signedPrecommits, grandpa.SignedPrecommit[Hash, N, string, primitives.AuthorityID]{ Precommit: pc.Precommit, Signature: string(pc.Signature[:]), - ID: string(pc.ID.Bytes()), + ID: (pc.ID), }) } - commitValidationResult, err := grandpa.ValidateCommit[Hash, N, string, string]( - grandpa.Commit[Hash, N, string, string]{ + commitValidationResult, err := grandpa.ValidateCommit[Hash, N, string, primitives.AuthorityID]( + grandpa.Commit[Hash, N, string, primitives.AuthorityID]{ TargetHash: j.Justification.Commit.TargetHash, TargetNumber: j.Justification.Commit.TargetNumber, Precommits: signedPrecommits, diff --git a/internal/client/consensus/grandpa/justification_test.go b/internal/client/consensus/grandpa/justification_test.go index 305ba1b6c1..48fa61c0e2 100644 --- a/internal/client/consensus/grandpa/justification_test.go +++ b/internal/client/consensus/grandpa/justification_test.go @@ -57,7 +57,7 @@ func TestJustificationEncoding(t *testing.T) { hash.H256(""), hash.H256(""), hash.H256(hashA), - runtime.Digest{}), + runtime.Digest{Logs: []runtime.DigestItem{}}), ) expected := primitives.GrandpaJustification[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]{ @@ -106,7 +106,7 @@ func TestDecodeGrandpaJustificationVerifyFinalizes(t *testing.T) { invalidEncoding, HashNumber[hash.H256, uint64]{}, 2, - grandpa.VoterSet[string]{}) + grandpa.VoterSet[primitives.AuthorityID]{}) require.Error(t, err) // Invalid target @@ -127,7 +127,7 @@ func TestDecodeGrandpaJustificationVerifyFinalizes(t *testing.T) { encWrongTarget, HashNumber[hash.H256, uint64]{}, 2, - grandpa.VoterSet[string]{}) + grandpa.VoterSet[primitives.AuthorityID]{}) require.Error(t, err) require.ErrorContains(t, err, "invalid commit target in grandpa justification") @@ -136,7 +136,7 @@ func TestDecodeGrandpaJustificationVerifyFinalizes(t *testing.T) { hash.H256(""), hash.H256(""), a, - runtime.Digest{}) + runtime.Digest{Logs: []runtime.DigestItem{}}) headerList := []generic.Header[uint64, hash.H256, runtime.BlakeTwo256]{*headerB} @@ -165,7 +165,7 @@ func TestDecodeGrandpaJustificationVerifyFinalizes(t *testing.T) { Number: 1, } - idWeights := make([]grandpa.IDWeight[string], 0) + idWeights := make([]grandpa.IDWeight[primitives.AuthorityID], 0) for i := 1; i <= 4; i++ { var id ced25519.Public switch i { @@ -178,8 +178,8 @@ func TestDecodeGrandpaJustificationVerifyFinalizes(t *testing.T) { case 4: id = ed25519.Ferdie.Pair().Public().(ced25519.Public) } - idWeights = append(idWeights, grandpa.IDWeight[string]{ - ID: string(id[:]), Weight: 1, + idWeights = append(idWeights, grandpa.IDWeight[primitives.AuthorityID]{ + ID: primitives.AuthorityID(id[:]), Weight: 1, }) } voters := grandpa.NewVoterSet(idWeights) @@ -258,7 +258,7 @@ func TestJustification_verify(t *testing.T) { func TestJustification_verifyWithVoterSet(t *testing.T) { // 1) invalid commit - idWeights := make([]grandpa.IDWeight[string], 0) + idWeights := make([]grandpa.IDWeight[primitives.AuthorityID], 0) for i := 1; i <= 4; i++ { var id ced25519.Public switch i { @@ -271,8 +271,8 @@ func TestJustification_verifyWithVoterSet(t *testing.T) { case 4: id = ed25519.Ferdie.Pair().Public().(ced25519.Public) } - idWeights = append(idWeights, grandpa.IDWeight[string]{ - ID: string(id[:]), Weight: 1, + idWeights = append(idWeights, grandpa.IDWeight[primitives.AuthorityID]{ + ID: primitives.AuthorityID(id[:]), Weight: 1, }) } voters := grandpa.NewVoterSet(idWeights) diff --git a/internal/client/consensus/grandpa/notification.go b/internal/client/consensus/grandpa/notification.go index 62607287a0..2c7782163a 100644 --- a/internal/client/consensus/grandpa/notification.go +++ b/internal/client/consensus/grandpa/notification.go @@ -14,3 +14,16 @@ import ( type GrandpaJustificationSender[Hash runtime.Hash, N runtime.Number, Header runtime.Header[N, Hash]] struct { notification.NotificationSender[GrandpaJustification[Hash, N, Header]] } + +// / The receiving half of the Grandpa justification channel. +// / +// / Used to receive notifications about justifications generated +// / at the end of a Grandpa round. +// / The `GrandpaJustificationStream` entity stores the `SharedJustificationSenders` +// / so it can be used to add more subscriptions. +// pub type GrandpaJustificationStream = +// +// NotificationStream, GrandpaJustificationsTracingKey>; +type GrandpaJustificationStream[Hash runtime.Hash, N runtime.Number, Header runtime.Header[N, Hash]] struct { + notification.NotificationStream[GrandpaJustification[Hash, N, Header]] +} diff --git a/internal/client/consensus/grandpa/until_imported.go b/internal/client/consensus/grandpa/until_imported.go index f914062f7d..80fea6ef94 100644 --- a/internal/client/consensus/grandpa/until_imported.go +++ b/internal/client/consensus/grandpa/until_imported.go @@ -4,11 +4,13 @@ package grandpa import ( + "sync" "time" "github.com/ChainSafe/gossamer/internal/client/api" primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" "github.com/ChainSafe/gossamer/internal/primitives/runtime" + grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" "github.com/gammazero/deque" ) @@ -40,7 +42,8 @@ type discard struct{} func (discard) isDiscardWaitOrReady() {} -type wait[H, N, M any] []struct { +type wait[H, N, M any] []waitItem[H, N, M] +type waitItem[H, N, M any] struct { TargetHash H TargetNumber N Wait M @@ -75,6 +78,7 @@ type untilImported[ // Queue identifier for differentiation in logs. identifier string // TODO: metrics + constructor func() M } type pendingEntry[H runtime.Hash, N runtime.Number, Blocked any] struct { BlockNumber N @@ -94,12 +98,13 @@ func newUntilImported[ statusCheck BlockStatus[H, N], incomingMessages <-chan Blocked, identifier string, + constructor func() M, ) untilImported[H, N, Header, Blocked, M] { // how often to check if pending messages that are waiting for blocks to be imported can be checked. // // the import notifications interval takes care of most of this; this is used in the event of missed // import notifications - const checkPendingInterval = 5 * time.Second + const checkPendingInterval = 1 * time.Second checkPending := time.NewTicker(checkPendingInterval).C @@ -112,20 +117,30 @@ func newUntilImported[ checkPending: checkPending, pending: make(map[H]pendingEntry[H, N, Blocked]), identifier: identifier, + constructor: constructor, } } -func (ui *untilImported[H, N, Header, Blocked, M]) Chan() <-chan Blocked { - ch := make(chan Blocked) +type blockedError[Blocked any] struct { + Blocked Blocked + Error error +} + +func (ui *untilImported[H, N, Header, Blocked, M]) Chan() <-chan blockedError[Blocked] { + ch := make(chan blockedError[Blocked]) go func() { defer close(ch) for { ready, blocked, err := ui.pollNext() - if err != nil { - return - } - if ready && blocked != nil { - ch <- *blocked + if ready { + if err != nil { + ch <- blockedError[Blocked]{Error: err} + } else if blocked != nil { + ch <- blockedError[Blocked]{Blocked: *blocked} + } else { + // close channel, no more messages to process. + return + } } } }() @@ -141,7 +156,7 @@ incoming: return true, nil, nil } // new input: schedule wait of any parts which require blocks to be known. - dwr, err := (*new(M)).NeedsWaiting(b, ui.statusCheck) + dwr, err := ui.constructor().NeedsWaiting(b, ui.statusCheck) if err != nil { return true, nil, err } @@ -208,7 +223,6 @@ imports: if updateInterval { knownKeys := make([]HashNumber[H, N], 0) for blockHash, e := range ui.pending { - number, err := ui.statusCheck.Number(blockHash) if err != nil { return true, nil, err @@ -217,7 +231,8 @@ imports: knownKeys = append(knownKeys, HashNumber[H, N]{Hash: blockHash, Number: *number}) } else { nextLog := e.LastLog.Add(logPendingInterval) - if time.Now().After(nextLog) { + now := time.Now() + if now.After(nextLog) || now.Equal(nextLog) { logger.Debugf( "Waiting to import block %s before %d %s messages can be imported. "+ "Requesting network sync service to retrieve block from. Possible fork?", @@ -330,6 +345,327 @@ func newUntilVoteTargetImported[H runtime.Hash, N runtime.Number, Header runtime statusCheck, incomingMessages, identifier, + func() signedMessage[H, N] { + return signedMessage[H, N]{} + }, ) return untilVoteTargetImported[H, N, Header]{uvti} } + +type refCount[T any] struct { + inner *T + count int + sync.Mutex +} + +// / This blocks a global message import, i.e. a commit or catch up messages, +// / until all blocks referenced in its votes are known. +// / +// / This is used for compact commits and catch up messages which have already +// / been checked for structural soundness (e.g. valid signatures). +// / +// / We use the `Arc`'s reference count to implicitly count the number of outstanding blocks that we +// / are waiting on for the same message (i.e. other `BlockGlobalMessage` instances with the same +// / `inner`). +// +// pub(crate) struct BlockGlobalMessage { +// inner: Arc>>>, +// target_number: NumberFor, +// } +type blockGlobalMessage[H runtime.Hash, N runtime.Number] struct { + inner *refCount[communicationIn[H, N]] + targetNumber N +} + +type known[N runtime.Number] struct { + number N +} +type unknown[N runtime.Number] struct { + number N +} + +func (k known[N]) Number() N { return k.number } +func (u unknown[N]) Number() N { return u.number } + +type knownOrUnknown[N runtime.Number] interface { + Number() N +} + +type itemToAwait[H runtime.Hash, N runtime.Number] struct { + targetHash H + targetNumber N + blockGlobalMessage[H, N] +} + +// impl Unpin for BlockGlobalMessage {} + +// impl BlockUntilImported for BlockGlobalMessage { +// type Blocked = CommunicationIn; + +// fn needs_waiting>( +// +// input: Self::Blocked, +// status_check: &BlockStatus, +// +// ) -> Result, Error> { +func (bgm blockGlobalMessage[H, N]) NeedsWaiting( + input communicationIn[H, N], + statusCheck BlockStatus[H, N], +) (discardWaitOrReady, error) { + // let mut checked_hashes: HashMap<_, KnownOrUnknown>> = HashMap::new(); + checkedHashes := map[H]knownOrUnknown[N]{} + { + // returns false when should early exit. + // let mut query_known = |target_hash, perceived_number| -> Result { + var queryKnown = func(targetHash H, perceivedNumber N) (bool, error) { + // let canon_number = match checked_hashes.entry(target_hash) { + // Entry::Occupied(entry) => *entry.get().number(), + // Entry::Vacant(entry) => { + // if let Some(number) = status_check.block_number(target_hash)? { + // entry.insert(KnownOrUnknown::Known(number)); + // number + // } else { + // entry.insert(KnownOrUnknown::Unknown(perceived_number)); + // perceived_number + // } + // }, + // }; + + // check integrity: all votes for same hash have same number. + var canonNumber N + entry, ok := checkedHashes[targetHash] + if ok { + canonNumber = entry.Number() + } else { + number, err := statusCheck.Number(targetHash) + if err != nil { + return false, err + } + if number != nil { + checkedHashes[targetHash] = known[N]{number: *number} + canonNumber = *number + } else { + checkedHashes[targetHash] = unknown[N]{number: perceivedNumber} + canonNumber = perceivedNumber + } + } + + // if canon_number != perceived_number { + // // invalid global message: messages targeting wrong number + // // or at least different from other vote in same global + // // message. + // return Ok(false) + // } + if canonNumber != perceivedNumber { + // invalid global message: messages targeting wrong number + // or at least different from other vote in same global + // message. + return false, nil + } + + return true, nil + } + + // match input { + switch input := input.(type) { + // voter::CommunicationIn::Commit(_, ref commit, ..) => { + case grandpa.CommunicationInCommit[H, N, primitives.AuthoritySignature, primitives.AuthorityID]: + // add known hashes from all precommits. + // let precommit_targets = + // commit.precommits.iter().map(|c| (c.target_number, c.target_hash)); + var precommitTargets []HashNumber[H, N] + for _, c := range input.CompactCommit.Precommits { + precommitTargets = append(precommitTargets, HashNumber[H, N]{ + Hash: c.TargetHash, + Number: c.TargetNumber, + }) + } + + // for (target_number, target_hash) in precommit_targets { + // if !query_known(target_hash, target_number)? { + // return Ok(DiscardWaitOrReady::Discard) + // } + // } + for _, target := range precommitTargets { + known, err := queryKnown(target.Hash, target.Number) + if err != nil { + return nil, err + } + if !known { + return discard{}, nil + } + } + // }, + // voter::CommunicationIn::CatchUp(ref catch_up, ..) => { + case grandpa.CommunicationInCatchUp[H, N, primitives.AuthoritySignature, primitives.AuthorityID]: + // add known hashes from all prevotes and precommits. + // let prevote_targets = catch_up + // .prevotes + // .iter() + // .map(|s| (s.prevote.target_number, s.prevote.target_hash)); + prevoteTargets := make([]HashNumber[H, N], len(input.CatchUp.Prevotes)) + for i, s := range input.CatchUp.Prevotes { + prevoteTargets[i] = HashNumber[H, N]{ + Hash: s.Prevote.TargetHash, + Number: s.Prevote.TargetNumber, + } + } + + // let precommit_targets = catch_up + // .precommits + // .iter() + // .map(|s| (s.precommit.target_number, s.precommit.target_hash)); + precommitTargets := make([]HashNumber[H, N], len(input.CatchUp.Precommits)) + for i, s := range input.CatchUp.Precommits { + precommitTargets[i] = HashNumber[H, N]{ + Hash: s.Precommit.TargetHash, + Number: s.Precommit.TargetNumber, + } + } + + // let targets = prevote_targets.chain(precommit_targets); + targets := append(prevoteTargets, precommitTargets...) + + // for (target_number, target_hash) in targets { + // if !query_known(target_hash, target_number)? { + // return Ok(DiscardWaitOrReady::Discard) + // } + // } + // }, + for _, target := range targets { + known, err := queryKnown(target.Hash, target.Number) + if err != nil { + return nil, err + } + if !known { + return discard{}, nil + } + } + default: + panic("unreachable") + } + } + + // let unknown_hashes = checked_hashes + // .into_iter() + // .filter_map(|(hash, num)| match num { + // KnownOrUnknown::Unknown(number) => Some((hash, number)), + // KnownOrUnknown::Known(_) => None, + // }) + // .collect::>(); + unknownHashes := make([]HashNumber[H, N], 0) + for hash, num := range checkedHashes { + switch num := num.(type) { + case unknown[N]: + unknownHashes = append(unknownHashes, HashNumber[H, N]{ + Hash: hash, + Number: num.Number(), + }) + case known[N]: + default: + panic("unreachable") + } + } + + // if unknown_hashes.is_empty() { + if len(unknownHashes) == 0 { + // none of the hashes in the global message were unknown. + // we can just return the message directly. + // return Ok(DiscardWaitOrReady::Ready(input)) + return ready[communicationIn[H, N]]{input}, nil + } + + // let locked_global = Arc::new(Mutex::new(Some(input))); + lockedGlobal := &refCount[communicationIn[H, N]]{inner: &input, count: 0} + + // let items_to_await = unknown_hashes + // .into_iter() + // .map(|(hash, target_number)| { + // ( + // hash, + // target_number, + // BlockGlobalMessage { inner: locked_global.clone(), target_number }, + // ) + // }) + // .collect(); + + itemsToAwait := make(wait[H, N, *blockGlobalMessage[H, N]], len(unknownHashes)) + for i, target := range unknownHashes { + lockedGlobal.count++ + itemsToAwait[i] = waitItem[H, N, *blockGlobalMessage[H, N]]{ + TargetHash: target.Hash, + TargetNumber: target.Number, + Wait: &blockGlobalMessage[H, N]{ + inner: lockedGlobal, + targetNumber: target.Number, + }, + } + } + + // schedule waits for all unknown messages. + // when the last one of these has `wait_completed` called on it, + // the global message will be returned. + // Ok(DiscardWaitOrReady::Wait(items_to_await)) + return itemsToAwait, nil +} + +// fn wait_completed(self, canon_number: NumberFor) -> Option { +func (bgm *blockGlobalMessage[H, N]) WaitCompleted(canonNumber N) *communicationIn[H, N] { + // if self.target_number != canon_number { + if bgm.targetNumber != canonNumber { + // Delete the inner message so it won't ever be forwarded. Future calls to + // `wait_completed` on the same `inner` will ignore it. + // *self.inner.lock() = None; + // return None + bgm.inner.Lock() + bgm.inner.inner = nil + bgm.inner.Unlock() + } + + // match Arc::try_unwrap(self.inner) { + bgm.inner.Lock() + defer bgm.inner.Unlock() + bgm.inner.count-- + if bgm.inner.count < 0 { + panic("unreachable") + } + if bgm.inner.count == 0 { + // // This is the last reference and thus the last outstanding block to be awaited. `inner` + // // is either `Some(_)` or `None`. The latter implies that a previous `wait_completed` + // // call witnessed a block number mismatch (see above). + // Ok(inner) => Mutex::into_inner(inner), + return bgm.inner.inner + } + // There are still other strong references to this `Arc`, thus the message is blocked on + // other blocks to be imported. + return nil +} + +// / A stream which gates off incoming global messages, i.e. commit and catch up +// / messages, until all referenced block hashes have been imported. +// pub(crate) type UntilGlobalMessageBlocksImported = +// +// UntilImported>; +type untilGlobalMessageBlocksImported[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]] struct { + untilImported[H, N, Header, communicationIn[H, N], *blockGlobalMessage[H, N]] +} + +func newUntilGlobalMessageBlocksImported[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]]( + importNotifications <-chan api.BlockImportNotification[H, N, Header], + blockSyncRequester BlockSyncRequester[H, N], + statusCheck BlockStatus[H, N], + incomingMessages <-chan communicationIn[H, N], + identifier string, +) untilGlobalMessageBlocksImported[H, N, Header] { + ui := newUntilImported[H, N, Header, communicationIn[H, N], *blockGlobalMessage[H, N]]( + importNotifications, + blockSyncRequester, + statusCheck, + incomingMessages, + identifier, + func() *blockGlobalMessage[H, N] { + return &blockGlobalMessage[H, N]{} + }, + ) + return untilGlobalMessageBlocksImported[H, N, Header]{ui} +} diff --git a/internal/client/consensus/grandpa/until_imported_test.go b/internal/client/consensus/grandpa/until_imported_test.go new file mode 100644 index 0000000000..c466ca0de1 --- /dev/null +++ b/internal/client/consensus/grandpa/until_imported_test.go @@ -0,0 +1,525 @@ +package grandpa + +import ( + "bytes" + "slices" + "sync" + "testing" + "time" + + "github.com/ChainSafe/gossamer/internal/client/api" + peerid "github.com/ChainSafe/gossamer/internal/client/network/types/peer-id" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/common" + primitives "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" + "github.com/ChainSafe/gossamer/internal/primitives/core/hash" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/runtime/generic" + grandpa "github.com/ChainSafe/gossamer/pkg/finality-grandpa" + "github.com/stretchr/testify/require" +) + +type TestChainState[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]] struct { + sender chan api.BlockImportNotification[H, N, Header] + knownBlocks map[H]N + knownBlocksMtx sync.Mutex +} + +func NewTestChainState[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]]() (*TestChainState[H, N, Header], api.ImportNotifications[H, N, Header]) { + ch := make(api.ImportNotifications[H, N, Header], 100000) + return &TestChainState[H, N, Header]{ + sender: ch, + knownBlocks: make(map[H]N), + }, ch +} + +func (t *TestChainState[H, N, Header]) BlockStatus() TestBlockStatus[H, N] { + return TestBlockStatus[H, N]{ + inner: &t.knownBlocks, + innerMtx: &t.knownBlocksMtx, + } +} + +func (t *TestChainState[H, N, Header]) ImportHeader(header Header) { + hash := header.Hash() + number := header.Number() + t.knownBlocksMtx.Lock() + t.knownBlocks[hash] = number + t.knownBlocksMtx.Unlock() + + t.sender <- api.BlockImportNotification[H, N, Header]{ + Hash: hash, + Origin: common.FileBlockOrigin, + Header: header, + IsNewBest: false, + TreeRoute: nil, + } +} + +type TestBlockStatus[H runtime.Hash, N runtime.Number] struct { + inner *map[H]N + innerMtx *sync.Mutex +} + +func (tbs TestBlockStatus[H, N]) Number(hash H) (*N, error) { + num, ok := (*tbs.inner)[hash] + if !ok { + return nil, nil + } + return &num, nil +} + +type TestBlockSyncRequester[H runtime.Hash, N runtime.Number] struct { + requests []HashNumber[H, N] + requestsMtx sync.Mutex +} + +func (tbsr *TestBlockSyncRequester[H, N]) SetSyncForkRequest(peers []peerid.PeerID, hash H, number N) { + tbsr.requestsMtx.Lock() + defer tbsr.requestsMtx.Unlock() + tbsr.requests = append(tbsr.requests, HashNumber[H, N]{Hash: hash, Number: number}) +} + +func makeHeader(number uint64) *generic.Header[uint64, hash.H256, runtime.BlakeTwo256] { + return generic.NewHeader[uint64, hash.H256, runtime.BlakeTwo256]( + number, + "", + "", + "", + runtime.Digest{}, + ) +} + +type numCompactCommit[H runtime.Hash, N runtime.Number] struct { + Number uint64 + primitives.CompactCommit[H, N] +} + +// unwrap the commit from `CommunicationIn` returning its fields in a tuple, +// panics if the given message isn't a commit +func unapplyCommit[H runtime.Hash, N runtime.Number](msg communicationIn[H, N]) numCompactCommit[H, N] { + switch msg := msg.(type) { + case grandpa.CommunicationInCommit[H, N, primitives.AuthoritySignature, primitives.AuthorityID]: + return numCompactCommit[H, N]{ + Number: msg.Number, + CompactCommit: primitives.CompactCommit[H, N](msg.CompactCommit), + } + default: + panic("expected commit") + } +} + +// unwrap the catch up from `CommunicationIn` returning its inner representation, +// panics if the given message isn't a catch up +func unapplyCatchUp[H runtime.Hash, N runtime.Number](msg communicationIn[H, N]) grandpa.CatchUp[H, N, primitives.AuthoritySignature, primitives.AuthorityID] { + switch msg := msg.(type) { + case grandpa.CommunicationInCatchUp[H, N, primitives.AuthoritySignature, primitives.AuthorityID]: + return msg.CatchUp + default: + panic("expected catch up") + } +} + +func messageAllDependenciesSatisfied( + t *testing.T, + msg communicationIn[hash.H256, uint64], + enactDependencies func(*TestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]), +) communicationIn[hash.H256, uint64] { + chainState, importNotifications := NewTestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]() + blockStatus := chainState.BlockStatus() + + // enact all dependencies before importing the message + enactDependencies(chainState) + + global := make(chan communicationIn[hash.H256, uint64], 100000) + + untilImported := newUntilGlobalMessageBlocksImported( + importNotifications, + &TestBlockSyncRequester[hash.H256, uint64]{}, + blockStatus, + global, + "global", + ) + + global <- msg + + untilImportedChan := untilImported.Chan() + + be := <-untilImportedChan + require.NoError(t, be.Error) + require.NotNil(t, be.Blocked) + + return be.Blocked +} + +func blockingMessageOnDependencies( + t *testing.T, + msg communicationIn[hash.H256, uint64], + enactDependencies func(*TestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]), +) communicationIn[hash.H256, uint64] { + chainState, importNotifications := NewTestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]() + blockStatus := chainState.BlockStatus() + + global := make(chan communicationIn[hash.H256, uint64], 100000) + + untilImported := newUntilGlobalMessageBlocksImported( + importNotifications, + &TestBlockSyncRequester[hash.H256, uint64]{}, + blockStatus, + global, + "global", + ) + + global <- msg + + timeout := time.NewTimer(100 * time.Millisecond) + var timeoutFired bool + untilImportedChan := untilImported.Chan() + for { + select { + case <-timeout.C: + // timeout fired. push in the headers. + timeoutFired = true + enactDependencies(chainState) + case be := <-untilImportedChan: + if !timeoutFired { + t.Fatalf("timeout should have fired first") + } + require.NoError(t, be.Error) + require.NotNil(t, be.Blocked) + + return be.Blocked + } + } +} + +func TestUntilImported(t *testing.T) { + t.Run("blocking_commit_message", func(t *testing.T) { + h1 := makeHeader(5) + h2 := makeHeader(6) + h3 := makeHeader(7) + + unknownCommit := primitives.CompactCommit[hash.H256, uint64]{ + TargetHash: h1.Hash(), + TargetNumber: 5, + Precommits: []grandpa.Precommit[hash.H256, uint64]{ + {TargetHash: h2.Hash(), TargetNumber: 6}, + {TargetHash: h3.Hash(), TargetNumber: 7}, + }, + AuthData: nil, // not used + } + + uc := grandpa.CommunicationInCommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + Number: 0, + CompactCommit: grandpa.CompactCommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID](unknownCommit), + Callback: func(cpo grandpa.CommitProcessingOutcome) {}, + } + + res := blockingMessageOnDependencies(t, uc, func(chainState *TestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]) { + chainState.ImportHeader(*h1) + chainState.ImportHeader(*h2) + chainState.ImportHeader(*h3) + }) + + require.Equal(t, unapplyCommit[hash.H256, uint64](res), unapplyCommit[hash.H256, uint64](uc)) + }) + + t.Run("commit_message_all_known", func(t *testing.T) { + h1 := makeHeader(5) + h2 := makeHeader(6) + h3 := makeHeader(7) + + knownCommit := primitives.CompactCommit[hash.H256, uint64]{ + TargetHash: h1.Hash(), + TargetNumber: 5, + Precommits: []grandpa.Precommit[hash.H256, uint64]{ + {TargetHash: h2.Hash(), TargetNumber: 6}, + {TargetHash: h3.Hash(), TargetNumber: 7}, + }, + AuthData: nil, // not used + } + + kc := grandpa.CommunicationInCommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + Number: 0, + CompactCommit: grandpa.CompactCommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID](knownCommit), + Callback: func(cpo grandpa.CommitProcessingOutcome) {}, + } + + res := blockingMessageOnDependencies(t, kc, func(chainState *TestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]) { + chainState.ImportHeader(*h1) + chainState.ImportHeader(*h2) + chainState.ImportHeader(*h3) + }) + + require.Equal(t, unapplyCommit[hash.H256, uint64](res), unapplyCommit[hash.H256, uint64](kc)) + }) + + t.Run("blocking_catch_up_message", func(t *testing.T) { + h1 := makeHeader(5) + h2 := makeHeader(6) + h3 := makeHeader(7) + + signedPrevote := func(header *generic.Header[uint64, hash.H256, runtime.BlakeTwo256]) grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID] { + return grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + ID: primitives.AuthorityID(bytes.Repeat([]byte{1}, 32)), + Signature: primitives.AuthoritySignature(bytes.Repeat([]byte{1}, 64)), + Prevote: grandpa.Prevote[hash.H256, uint64]{ + TargetHash: header.Hash(), + TargetNumber: header.Number(), + }, + } + } + + signedPrecommit := func(header *generic.Header[uint64, hash.H256, runtime.BlakeTwo256]) grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID] { + return grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + ID: primitives.AuthorityID(bytes.Repeat([]byte{1}, 32)), + Signature: primitives.AuthoritySignature(bytes.Repeat([]byte{1}, 64)), + Precommit: grandpa.Precommit[hash.H256, uint64]{ + TargetHash: header.Hash(), + TargetNumber: header.Number(), + }, + } + } + + prevotes := []grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + signedPrevote(h1), + signedPrevote(h3), + } + + precommits := []grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + signedPrecommit(h1), + signedPrecommit(h2), + } + + unknownCatchUp := grandpa.CatchUp[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + RoundNumber: 1, + Prevotes: prevotes, + Precommits: precommits, + BaseHash: h1.Hash(), + BaseNumber: h1.Number(), + } + + uc := grandpa.CommunicationInCatchUp[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + CatchUp: unknownCatchUp, + Callback: func(cpo grandpa.CatchUpProcessingOutcome) {}, + } + + res := blockingMessageOnDependencies(t, uc, func(chainState *TestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]) { + chainState.ImportHeader(*h1) + chainState.ImportHeader(*h2) + chainState.ImportHeader(*h3) + }) + + require.Equal(t, unapplyCatchUp[hash.H256, uint64](res), unapplyCatchUp[hash.H256, uint64](uc)) + }) + + t.Run("catch_up_message_all_known", func(t *testing.T) { + h1 := makeHeader(5) + h2 := makeHeader(6) + h3 := makeHeader(7) + + signedPrevote := func(header *generic.Header[uint64, hash.H256, runtime.BlakeTwo256]) grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID] { + return grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + ID: primitives.AuthorityID(bytes.Repeat([]byte{1}, 32)), + Signature: primitives.AuthoritySignature(bytes.Repeat([]byte{1}, 64)), + Prevote: grandpa.Prevote[hash.H256, uint64]{ + TargetHash: header.Hash(), + TargetNumber: header.Number(), + }, + } + } + + signedPrecommit := func(header *generic.Header[uint64, hash.H256, runtime.BlakeTwo256]) grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID] { + return grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + ID: primitives.AuthorityID(bytes.Repeat([]byte{1}, 32)), + Signature: primitives.AuthoritySignature(bytes.Repeat([]byte{1}, 64)), + Precommit: grandpa.Precommit[hash.H256, uint64]{ + TargetHash: header.Hash(), + TargetNumber: header.Number(), + }, + } + } + + prevotes := []grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + signedPrevote(h1), + signedPrevote(h3), + } + + precommits := []grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + signedPrecommit(h1), + signedPrecommit(h2), + } + + unknownCatchUp := grandpa.CatchUp[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + RoundNumber: 1, + Prevotes: prevotes, + Precommits: precommits, + BaseHash: h1.Hash(), + BaseNumber: h1.Number(), + } + + uc := grandpa.CommunicationInCatchUp[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + CatchUp: unknownCatchUp, + Callback: func(cpo grandpa.CatchUpProcessingOutcome) {}, + } + + res := messageAllDependenciesSatisfied(t, uc, func(chainState *TestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]) { + chainState.ImportHeader(*h1) + chainState.ImportHeader(*h2) + chainState.ImportHeader(*h3) + }) + + require.Equal(t, unapplyCatchUp[hash.H256, uint64](res), unapplyCatchUp[hash.H256, uint64](uc)) + }) + + t.Run("request_block_sync_for_needed_blocks", func(t *testing.T) { + chainState, importNotifications := NewTestChainState[hash.H256, uint64, generic.Header[uint64, hash.H256, runtime.BlakeTwo256]]() + blockStatus := chainState.BlockStatus() + + global := make(chan communicationIn[hash.H256, uint64], 100000) + + blockSyncRequester := &TestBlockSyncRequester[hash.H256, uint64]{} + + untilImported := newUntilGlobalMessageBlocksImported( + importNotifications, + blockSyncRequester, + blockStatus, + global, + "global", + ) + + h1 := makeHeader(5) + h2 := makeHeader(6) + h3 := makeHeader(7) + + // we create a commit message, with precommits for blocks 6 and 7 which + // we haven't imported. + unknownCommit := primitives.CompactCommit[hash.H256, uint64]{ + TargetHash: h1.Hash(), + TargetNumber: 5, + Precommits: []grandpa.Precommit[hash.H256, uint64]{ + {TargetHash: h2.Hash(), TargetNumber: 6}, + {TargetHash: h3.Hash(), TargetNumber: 7}, + }, + AuthData: nil, // not used + } + + uc := grandpa.CommunicationInCommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + Number: 0, + CompactCommit: grandpa.CompactCommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID](unknownCommit), + Callback: func(cpo grandpa.CommitProcessingOutcome) {}, + } + + // we send the commit message and spawn the until_imported stream + global <- uc + go func() { + for range untilImported.Chan() { + // consume the channel + } + }() + + // assert that we will make sync requests + assert := make(chan struct{}) + go func() { + for { + blockSyncRequester.requestsMtx.Lock() + blockSyncRequests := blockSyncRequester.requests + blockSyncRequester.requestsMtx.Unlock() + + // we request blocks targeted by the precommits that aren't imported + containsH2 := slices.ContainsFunc(blockSyncRequests, func(block HashNumber[hash.H256, uint64]) bool { + return block.Hash == h2.Hash() && block.Number == h2.Number() + }) + containsH3 := slices.ContainsFunc(blockSyncRequests, func(block HashNumber[hash.H256, uint64]) bool { + return block.Hash == h3.Hash() && block.Number == h3.Number() + }) + + if containsH2 && containsH3 { + close(assert) + return + } + } + }() + + // the `until_imported` stream doesn't request the blocks immediately, + // but it should request them after a small timeout + timeout := time.NewTimer(60 * time.Second) + select { + case <-assert: + // success + case <-timeout.C: + t.Fatalf("timed out waiting for block sync request") + } + }) + + t.Run("block_global_message_wait_completed_return_when_all_awaited", func(t *testing.T) { + msg := testCatchUp() + inner := &refCount[communicationIn[hash.H256, uint64]]{ + inner: &msg, + } + + inner.count++ + waitingBlock1 := blockGlobalMessage[hash.H256, uint64]{ + inner: inner, + targetNumber: 1, + } + + inner.count++ + waitingBlock2 := blockGlobalMessage[hash.H256, uint64]{ + inner: inner, + targetNumber: 2, + } + + // waiting_block_2 is still waiting for block 2, thus this should return `None`. + require.Nil(t, waitingBlock1.WaitCompleted(1)) + + // Message only depended on block 1 and 2. Both have been imported, thus this should yield + // the message. + require.NotNil(t, waitingBlock2.WaitCompleted(2)) + }) + + t.Run("block_global_message_wait_completed_return_none_on_block_number_mismatch", func(t *testing.T) { + msg := testCatchUp() + inner := &refCount[communicationIn[hash.H256, uint64]]{ + inner: &msg, + } + + inner.count++ + waitingBlock1 := blockGlobalMessage[hash.H256, uint64]{ + inner: inner, + targetNumber: 1, + } + + inner.count++ + waitingBlock2 := blockGlobalMessage[hash.H256, uint64]{ + inner: inner, + targetNumber: 2, + } + + // Calling wait_completed with wrong block number should yield None. + require.Nil(t, waitingBlock1.WaitCompleted(1234)) + + // All blocks, that the message depended on, have been imported. Still, given the above + // block number mismatch this should return None. + require.Nil(t, waitingBlock2.WaitCompleted(2)) + }) + +} + +func testCatchUp() communicationIn[hash.H256, uint64] { + header := makeHeader(5) + + unknownCatchUp := grandpa.CatchUp[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + RoundNumber: 1, + Precommits: []grandpa.SignedPrecommit[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{}, + Prevotes: []grandpa.SignedPrevote[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{}, + BaseHash: header.Hash(), + BaseNumber: header.Number(), + } + + catchUp := grandpa.CommunicationInCatchUp[hash.H256, uint64, primitives.AuthoritySignature, primitives.AuthorityID]{ + CatchUp: unknownCatchUp, + Callback: func(cpo grandpa.CatchUpProcessingOutcome) {}, + } + + return catchUp +} diff --git a/internal/client/db/backend.go b/internal/client/db/backend.go index 23bd53c6bb..097d8c4a7b 100644 --- a/internal/client/db/backend.go +++ b/internal/client/db/backend.go @@ -16,6 +16,7 @@ import ( "github.com/ChainSafe/gossamer/internal/client/db/offchain" statedb "github.com/ChainSafe/gossamer/internal/client/state-db" hashdb "github.com/ChainSafe/gossamer/internal/hash-db" + memorykvdb "github.com/ChainSafe/gossamer/internal/kvdb/memory-kvdb" "github.com/ChainSafe/gossamer/internal/log" memorydb "github.com/ChainSafe/gossamer/internal/memory-db" "github.com/ChainSafe/gossamer/internal/primitives/blockchain" @@ -370,8 +371,7 @@ type emptyStorage[H runtime.Hash] struct { func newEmptyStorage[H runtime.Hash, Hasher runtime.Hasher[H]]() emptyStorage[H] { var root H mdb := trie.NewMemoryDB[H, Hasher]() - trie := triedb.NewEmptyTrieDB[H, Hasher](mdb) - trie.SetVersion(triedb.V1) + trie := triedb.NewEmptyTrieDB[H, Hasher](mdb, trie.LayoutV1[Hasher, H]{}) root = trie.MustHash() return emptyStorage[H]{root} } @@ -435,6 +435,78 @@ func NewBackend[ return newBackendFromDatabase[H, N, E, Hasher, Header](db, canonicalizationDelay, dbConfig, needsInit) } +// / Create new memory-backed client backend for tests. +// #[cfg(any(test, feature = "test-helpers"))] +// +// pub fn new_test(blocks_pruning: u32, canonicalization_delay: u64) -> Self { +// Self::new_test_with_tx_storage(BlocksPruning::Some(blocks_pruning), canonicalization_delay) +// } +func NewTestBackend[ + H runtime.Hash, + N runtime.Number, + E runtime.Extrinsic, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], +](blocksPruning uint32, canonicalizationDelay uint64) *Backend[H, Hasher, N, E, Header] { + return NewTestBackendWithTxStorage[H, N, E, Hasher, Header](BlocksPruningSome(blocksPruning), canonicalizationDelay) +} + +/// Create new memory-backed client backend for tests. +// #[cfg(any(test, feature = "test-helpers"))] +// pub fn new_test_with_tx_storage( +// blocks_pruning: BlocksPruning, +// canonicalization_delay: u64, +// ) -> Self { +// let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); +// let db = sp_database::as_database(db); +// let state_pruning = match blocks_pruning { +// BlocksPruning::KeepAll => PruningMode::ArchiveAll, +// BlocksPruning::KeepFinalized => PruningMode::ArchiveCanonical, +// BlocksPruning::Some(n) => PruningMode::blocks_pruning(n), +// }; +// let db_setting = DatabaseSettings { +// trie_cache_maximum_size: Some(16 * 1024 * 1024), +// state_pruning: Some(state_pruning), +// source: DatabaseSource::Custom { db, require_create_flag: true }, +// blocks_pruning, +// }; + +// Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") +// } +func NewTestBackendWithTxStorage[ + H runtime.Hash, + N runtime.Number, + E runtime.Extrinsic, + Hasher runtime.Hasher[H], + Header runtime.Header[N, H], +](blocksPruning BlocksPruning, canonicalizationDelay uint64) *Backend[H, Hasher, N, E, Header] { + kvdb := memorykvdb.New(13) + var statePruning statedb.PruningMode + switch blocksPruning := blocksPruning.(type) { + case BlocksPruningKeepAll: + statePruning = statedb.PruningModeArchiveAll{} + case BlocksPruningKeepFinalized: + statePruning = statedb.PruningModeArchiveCanonical{} + case BlocksPruningSome: + statePruning = statedb.NewPruningModeConstrained(uint32(blocksPruning)) + default: + panic("unreachable") + } + trieCacheMaxSize := uint(16 * 1024 * 1024) + dbSetting := DatabaseConfig{ + TrieCacheMaximumSize: &trieCacheMaxSize, + StatePruning: statePruning, + Source: DatabaseSource{DB: database.NewDBAdapter[hash.H256](kvdb), RequireCreateFlag: true}, + BlocksPruning: blocksPruning, + } + + backend, err := NewBackend[H, N, E, Hasher, Header](dbSetting, canonicalizationDelay) + if err != nil { + panic("failed to create test-db") + } + return backend +} + func newBackendFromDatabase[ H runtime.Hash, N runtime.Number, diff --git a/internal/client/db/backend_test.go b/internal/client/db/backend_test.go index e41d88b6e4..c128080425 100644 --- a/internal/client/db/backend_test.go +++ b/internal/client/db/backend_test.go @@ -43,7 +43,7 @@ var ( ]{} ) -func NewTestBackend(t *testing.T, +func newTestBackend(t *testing.T, blocksPruning BlocksPruning, canonicalizationDelay uint64, ) *Backend[ hash.H256, @@ -211,7 +211,7 @@ func TestBackend(t *testing.T) { t.Run("block_hash_inserted_correctly", func(t *testing.T) { var backing database.Database[hash.H256] { - db := NewTestBackend(t, BlocksPruningSome(1), 0) + db := newTestBackend(t, BlocksPruningSome(1), 0) for i := uint64(0); i < 10; i++ { h, err := db.Blockchain().Hash(i) require.NoError(t, err) @@ -275,7 +275,7 @@ func TestBackend(t *testing.T) { t.Run("set_state_data", func(t *testing.T) { for i, stateVersion := range []storage.StateVersion{storage.StateVersionV0, storage.StateVersionV1} { t.Run(fmt.Sprintf("StateVersion%d", i), func(t *testing.T) { - db := NewTestBackend(t, BlocksPruningSome(2), 0) + db := newTestBackend(t, BlocksPruningSome(2), 0) var hash dbHash { op := db.beginOperation() @@ -379,7 +379,7 @@ func TestBackend(t *testing.T) { t.Run("delete_only_when_negative_rc", func(t *testing.T) { stateVersion := storage.StateVersionV1 var key dbHash - backend := NewTestBackend(t, BlocksPruningSome(1), 0) + backend := newTestBackend(t, BlocksPruningSome(1), 0) var hash dbHash { @@ -556,7 +556,7 @@ func TestBackend(t *testing.T) { }) t.Run("tree_route_works", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1000), 100) + backend := newTestBackend(t, BlocksPruningSome(1000), 100) blockchain := backend.blockchain block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) @@ -614,7 +614,7 @@ func TestBackend(t *testing.T) { }) t.Run("tree_route_child", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1000), 100) + backend := newTestBackend(t, BlocksPruningSome(1000), 100) blockchain := backend.blockchain block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) @@ -635,7 +635,7 @@ func TestBackend(t *testing.T) { }) t.Run("lowest_common_ancestor", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1000), 100) + backend := newTestBackend(t, BlocksPruningSome(1000), 100) blockchain := backend.blockchain block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) @@ -698,7 +698,7 @@ func TestBackend(t *testing.T) { }) t.Run("leaves_pruned_on_finality", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1a := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -735,7 +735,7 @@ func TestBackend(t *testing.T) { }) t.Run("test_aux", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(0), 0) + backend := newTestBackend(t, BlocksPruningSome(0), 0) val, err := backend.GetAux([]byte("test")) require.NoError(t, err) require.Nil(t, val) @@ -760,7 +760,7 @@ func TestBackend(t *testing.T) { copy(CON1EngineID[:], "CON1") t.Run("finalize_block_with_justification", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1 := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -778,7 +778,7 @@ func TestBackend(t *testing.T) { }) t.Run("append_justification_to_finalized_block", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1 := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -811,7 +811,7 @@ func TestBackend(t *testing.T) { }) t.Run("finalize_multiple_blocks_in_single_op", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1 := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -846,7 +846,7 @@ func TestBackend(t *testing.T) { t.Run("storage_hash_is_cached_correctly", func(t *testing.T) { stateVersion := storage.StateVersionV1 - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) var hash0 dbHash { @@ -956,7 +956,7 @@ func TestBackend(t *testing.T) { }) t.Run("finalize_non_sequential", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1 := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -977,7 +977,7 @@ func TestBackend(t *testing.T) { pruningModes := []BlocksPruning{BlocksPruningSome(2), BlocksPruningKeepFinalized{}, BlocksPruningKeepAll{}} for _, pruningMode := range pruningModes { - backend := NewTestBackend(t, pruningMode, 0) + backend := newTestBackend(t, pruningMode, 0) var blocks []hash.H256 var prevHash hash.H256 for i := 0; i < 5; i++ { @@ -1039,7 +1039,7 @@ func TestBackend(t *testing.T) { pruningModes := []BlocksPruning{BlocksPruningSome(2), BlocksPruningKeepFinalized{}, BlocksPruningKeepAll{}} for _, pruningMode := range pruningModes { - backend := NewTestBackend(t, pruningMode, 10) + backend := newTestBackend(t, pruningMode, 10) var blocks []hash.H256 var prevHash hash.H256 for i := 0; i < 5; i++ { @@ -1153,7 +1153,7 @@ func TestBackend(t *testing.T) { // 0 - 1b // \ - 1a - 2a - 3a // \ - 2b - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) var makeBlock = func(index uint64, parent hash.H256, val uint64) hash.H256 { hash, err := insertBlock(t, @@ -1225,7 +1225,7 @@ func TestBackend(t *testing.T) { }) t.Run("indexed_data_block_body", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1), 10) + backend := newTestBackend(t, BlocksPruningSome(1), 10) x0 := scale.MustMarshal(rt_testing.ExtrinsicsWrapper[uint64]{T: uint64(0)}) x1 := scale.MustMarshal(rt_testing.ExtrinsicsWrapper[uint64]{T: uint64(1)}) @@ -1287,7 +1287,7 @@ func TestBackend(t *testing.T) { }) t.Run("index_invalid_size", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1), 10) + backend := newTestBackend(t, BlocksPruningSome(1), 10) x0 := scale.MustMarshal(rt_testing.ExtrinsicsWrapper[uint64]{T: uint64(0)}) x1 := scale.MustMarshal(rt_testing.ExtrinsicsWrapper[uint64]{T: uint64(1)}) @@ -1325,7 +1325,7 @@ func TestBackend(t *testing.T) { }) t.Run("renew_transaction_storage", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(2), 10) + backend := newTestBackend(t, BlocksPruningSome(2), 10) var blocks []hash.H256 var prevHash hash.H256 x1 := scale.MustMarshal(rt_testing.ExtrinsicsWrapper[uint64]{T: uint64(0)}) @@ -1382,7 +1382,7 @@ func TestBackend(t *testing.T) { }) t.Run("remove_leaf_block", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(2), 10) + backend := newTestBackend(t, BlocksPruningSome(2), 10) var blocks []hash.H256 var prevHash hash.H256 for i := uint64(0); i < 2; i++ { @@ -1486,8 +1486,9 @@ func TestBackend(t *testing.T) { require.Equal(t, []hash.H256{bestHash}, children) }) + var layoutV1 = trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{} t.Run("import_existing_block_as_new_head", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 3) + backend := newTestBackend(t, BlocksPruningSome(10), 3) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1 := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -1500,9 +1501,8 @@ func TestBackend(t *testing.T) { // Insert 1 as best again. This should fail because canonicalization_delay == 3 // and best == 5 trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256]( - trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), + trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), layoutV1, ) - trie.SetVersion(triedb.V1) header := generic.NewHeader[uint64, dbHash, runtime.BlakeTwo256]( 1, dbHash(""), @@ -1531,7 +1531,7 @@ func TestBackend(t *testing.T) { }) t.Run("impoort_existing_state_fails", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) genesis, err := insertBlock(t, backend, 0, "", nil, "", nil, nil) require.NoError(t, err) @@ -1544,7 +1544,7 @@ func TestBackend(t *testing.T) { }) t.Run("leaves_not_created_for_ancient_blocks", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) @@ -1573,7 +1573,7 @@ func TestBackend(t *testing.T) { // attempt to revert 5 blocks. for _, pruningMode := range pruningModes { t.Run(fmt.Sprintf("%T", pruningMode), func(t *testing.T) { - backend := NewTestBackend(t, pruningMode, 1) + backend := newTestBackend(t, pruningMode, 1) var parent hash.H256 for i := uint64(0); i <= 10; i++ { @@ -1607,7 +1607,7 @@ func TestBackend(t *testing.T) { }) t.Run("revert_non_best_blocks", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) genesis, err := insertBlock(t, backend, 0, "", nil, "", nil, nil) require.NoError(t, err) @@ -1624,9 +1624,8 @@ func TestBackend(t *testing.T) { err := backend.BeginStateOperation(op, block1) require.NoError(t, err) trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256]( - trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), + trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), layoutV1, ) - trie.SetVersion(triedb.V1) header := generic.NewHeader[uint64, dbHash, runtime.BlakeTwo256]( 3, dbHash(""), @@ -1650,9 +1649,8 @@ func TestBackend(t *testing.T) { err := backend.BeginStateOperation(op, block2) require.NoError(t, err) trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256]( - trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), + trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), layoutV1, ) - trie.SetVersion(triedb.V1) header := generic.NewHeader[uint64, dbHash, runtime.BlakeTwo256]( 4, "", @@ -1676,9 +1674,8 @@ func TestBackend(t *testing.T) { err := backend.BeginStateOperation(op, block2) require.NoError(t, err) trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256]( - trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), + trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256](), layoutV1, ) - trie.SetVersion(triedb.V1) header := generic.NewHeader[uint64, dbHash, runtime.BlakeTwo256]( 3, hash.NewH256FromLowUint64BigEndian(42), @@ -1730,7 +1727,7 @@ func TestBackend(t *testing.T) { }) t.Run("no_duplicated_leaves_allowed", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(10), 10) + backend := newTestBackend(t, BlocksPruningSome(10), 10) block0 := insertHeader(t, backend, 0, hash.H256(""), nil, hash.H256("")) block1 := insertHeader(t, backend, 1, block0, nil, hash.H256("")) @@ -1758,7 +1755,7 @@ func TestBackend(t *testing.T) { for _, pruningMode := range pruningModes { t.Run(fmt.Sprintf("%T", pruningMode), func(t *testing.T) { - backend := NewTestBackend(t, pruningMode, 1) + backend := newTestBackend(t, pruningMode, 1) genesis, err := insertBlock(t, backend, 0, "", nil, "", nil, nil) require.NoError(t, err) @@ -1982,7 +1979,7 @@ func TestBackend(t *testing.T) { }) t.Run("pinned_blocks_on_finalize", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1), 10) + backend := newTestBackend(t, BlocksPruningSome(1), 10) var blocks []hash.H256 var prevHash hash.H256 @@ -2224,7 +2221,7 @@ func TestBackend(t *testing.T) { }) t.Run("pinned_blocks_on_finalize_with_fork", func(t *testing.T) { - backend := NewTestBackend(t, BlocksPruningSome(1), 10) + backend := newTestBackend(t, BlocksPruningSome(1), 10) var blocks []hash.H256 var prevHash hash.H256 diff --git a/internal/client/executor/executor.go b/internal/client/executor/executor.go index a8f0ef45ef..c9c5684f89 100644 --- a/internal/client/executor/executor.go +++ b/internal/client/executor/executor.go @@ -15,3 +15,367 @@ type RuntimeVersionOf interface { runtimeCode core.RuntimeCode, ) (version.RuntimeVersion, error) } + +// / An abstraction over Wasm code executor. Supports selecting execution backend and +// / manages runtime cache. +// pub struct WasmExecutor { +type WasmExecutor struct { + // /// Method used to execute fallback Wasm code. + // method: WasmExecutionMethod, + // /// The heap allocation strategy for onchain Wasm calls. + // default_onchain_heap_alloc_strategy: HeapAllocStrategy, + // /// The heap allocation strategy for offchain Wasm calls. + // default_offchain_heap_alloc_strategy: HeapAllocStrategy, + // /// Ignore onchain heap pages value. + // ignore_onchain_heap_pages: bool, + // /// WASM runtime cache. + // cache: Arc, + // /// The path to a directory which the executor can leverage for a file cache, e.g. put there + // /// compiled artifacts. + // cache_path: Option, + // /// Ignore missing function imports. + // allow_missing_host_functions: bool, + // phantom: PhantomData, +} + +// impl Clone for WasmExecutor { +// fn clone(&self) -> Self { +// Self { +// method: self.method, +// default_onchain_heap_alloc_strategy: self.default_onchain_heap_alloc_strategy, +// default_offchain_heap_alloc_strategy: self.default_offchain_heap_alloc_strategy, +// ignore_onchain_heap_pages: self.ignore_onchain_heap_pages, +// cache: self.cache.clone(), +// cache_path: self.cache_path.clone(), +// allow_missing_host_functions: self.allow_missing_host_functions, +// phantom: self.phantom, +// } +// } +// } + +// impl Default for WasmExecutor { +// fn default() -> Self { +// WasmExecutorBuilder::new().build() +// } +// } + +// impl WasmExecutor { +// /// Create new instance. +// /// +// /// # Parameters +// /// +// /// `method` - Method used to execute Wasm code. +// /// +// /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. Internally this +// /// will be mapped as [`HeapAllocStrategy::Static`] where `default_heap_pages` represent the +// /// static number of heap pages to allocate. Defaults to `DEFAULT_HEAP_ALLOC_STRATEGY` if `None` +// /// is provided. +// /// +// /// `max_runtime_instances` - The number of runtime instances to keep in memory ready for reuse. +// /// +// /// `cache_path` - A path to a directory where the executor can place its files for purposes of +// /// caching. This may be important in cases when there are many different modules with the +// /// compiled execution method is used. +// /// +// /// `runtime_cache_size` - The capacity of runtime cache. +// #[deprecated(note = "use `Self::builder` method instead of it")] +// pub fn new( +// method: WasmExecutionMethod, +// default_heap_pages: Option, +// max_runtime_instances: usize, +// cache_path: Option, +// runtime_cache_size: u8, +// ) -> Self { +// WasmExecutor { +// method, +// default_onchain_heap_alloc_strategy: unwrap_heap_pages( +// default_heap_pages.map(|h| HeapAllocStrategy::Static { extra_pages: h as _ }), +// ), +// default_offchain_heap_alloc_strategy: unwrap_heap_pages( +// default_heap_pages.map(|h| HeapAllocStrategy::Static { extra_pages: h as _ }), +// ), +// ignore_onchain_heap_pages: false, +// cache: Arc::new(RuntimeCache::new( +// max_runtime_instances, +// cache_path.clone(), +// runtime_cache_size, +// )), +// cache_path, +// allow_missing_host_functions: false, +// phantom: PhantomData, +// } +// } + +// /// Instantiate a builder for creating an instance of `Self`. +// pub fn builder() -> WasmExecutorBuilder { +// WasmExecutorBuilder::new() +// } + +// /// Ignore missing function imports if set true. +// #[deprecated(note = "use `Self::builder` method instead of it")] +// pub fn allow_missing_host_functions(&mut self, allow_missing_host_functions: bool) { +// self.allow_missing_host_functions = allow_missing_host_functions +// } +// } + +// impl WasmExecutor +// where +// H: HostFunctions, +// { +// /// Execute the given closure `f` with the latest runtime (based on `runtime_code`). +// /// +// /// The closure `f` is expected to return `Err(_)` when there happened a `panic!` in native code +// /// while executing the runtime in Wasm. If a `panic!` occurred, the runtime is invalidated to +// /// prevent any poisoned state. Native runtime execution does not need to report back +// /// any `panic!`. +// /// +// /// # Safety +// /// +// /// `runtime` and `ext` are given as `AssertUnwindSafe` to the closure. As described above, the +// /// runtime is invalidated on any `panic!` to prevent a poisoned state. `ext` is already +// /// implicitly handled as unwind safe, as we store it in a global variable while executing the +// /// native runtime. +// pub fn with_instance( +// &self, +// runtime_code: &RuntimeCode, +// ext: &mut dyn Externalities, +// heap_alloc_strategy: HeapAllocStrategy, +// f: F, +// ) -> Result +// where +// F: FnOnce( +// AssertUnwindSafe<&dyn WasmModule>, +// AssertUnwindSafe<&mut dyn WasmInstance>, +// Option<&RuntimeVersion>, +// AssertUnwindSafe<&mut dyn Externalities>, +// ) -> Result>, +// { +// match self.cache.with_instance::( +// runtime_code, +// ext, +// self.method, +// heap_alloc_strategy, +// self.allow_missing_host_functions, +// |module, instance, version, ext| { +// let module = AssertUnwindSafe(module); +// let instance = AssertUnwindSafe(instance); +// let ext = AssertUnwindSafe(ext); +// f(module, instance, version, ext) +// }, +// )? { +// Ok(r) => r, +// Err(e) => Err(e), +// } +// } + +// /// Perform a call into the given runtime. +// /// +// /// The runtime is passed as a [`RuntimeBlob`]. The runtime will be instantiated with the +// /// parameters this `WasmExecutor` was initialized with. +// /// +// /// In case of problems with during creation of the runtime or instantiation, a `Err` is +// /// returned. that describes the message. +// #[doc(hidden)] // We use this function for tests across multiple crates. +// pub fn uncached_call( +// &self, +// runtime_blob: RuntimeBlob, +// ext: &mut dyn Externalities, +// allow_missing_host_functions: bool, +// export_name: &str, +// call_data: &[u8], +// ) -> std::result::Result, Error> { +// self.uncached_call_impl( +// runtime_blob, +// ext, +// allow_missing_host_functions, +// export_name, +// call_data, +// &mut None, +// ) +// } + +// /// Same as `uncached_call`, except it also returns allocation statistics. +// #[doc(hidden)] // We use this function in tests. +// pub fn uncached_call_with_allocation_stats( +// &self, +// runtime_blob: RuntimeBlob, +// ext: &mut dyn Externalities, +// allow_missing_host_functions: bool, +// export_name: &str, +// call_data: &[u8], +// ) -> (std::result::Result, Error>, Option) { +// let mut allocation_stats = None; +// let result = self.uncached_call_impl( +// runtime_blob, +// ext, +// allow_missing_host_functions, +// export_name, +// call_data, +// &mut allocation_stats, +// ); +// (result, allocation_stats) +// } + +// fn uncached_call_impl( +// &self, +// runtime_blob: RuntimeBlob, +// ext: &mut dyn Externalities, +// allow_missing_host_functions: bool, +// export_name: &str, +// call_data: &[u8], +// allocation_stats_out: &mut Option, +// ) -> std::result::Result, Error> { +// let module = crate::wasm_runtime::create_wasm_runtime_with_code::( +// self.method, +// self.default_onchain_heap_alloc_strategy, +// runtime_blob, +// allow_missing_host_functions, +// self.cache_path.as_deref(), +// ) +// .map_err(|e| format!("Failed to create module: {}", e))?; + +// let instance = +// module.new_instance().map_err(|e| format!("Failed to create instance: {}", e))?; + +// let mut instance = AssertUnwindSafe(instance); +// let mut ext = AssertUnwindSafe(ext); +// let mut allocation_stats_out = AssertUnwindSafe(allocation_stats_out); + +// with_externalities_safe(&mut **ext, move || { +// let (result, allocation_stats) = +// instance.call_with_allocation_stats(export_name.into(), call_data); +// **allocation_stats_out = allocation_stats; +// result +// }) +// .and_then(|r| r) +// } +// } + +// impl sp_core::traits::ReadRuntimeVersion for WasmExecutor +// where +// H: HostFunctions, +// { +// fn read_runtime_version( +// &self, +// wasm_code: &[u8], +// ext: &mut dyn Externalities, +// ) -> std::result::Result, String> { +// let runtime_blob = RuntimeBlob::uncompress_if_needed(wasm_code) +// .map_err(|e| format!("Failed to create runtime blob: {:?}", e))?; + +// if let Some(version) = crate::wasm_runtime::read_embedded_version(&runtime_blob) +// .map_err(|e| format!("Failed to read the static section: {:?}", e)) +// .map(|v| v.map(|v| v.encode()))? +// { +// return Ok(version) +// } + +// // If the blob didn't have embedded runtime version section, we fallback to the legacy +// // way of fetching the version: i.e. instantiating the given instance and calling +// // `Core_version` on it. + +// self.uncached_call( +// runtime_blob, +// ext, +// // If a runtime upgrade introduces new host functions that are not provided by +// // the node, we should not fail at instantiation. Otherwise nodes that are +// // updated could run this successfully and it could lead to a storage root +// // mismatch when importing this block. +// true, +// "Core_version", +// &[], +// ) +// .map_err(|e| e.to_string()) +// } +// } + +// impl CodeExecutor for WasmExecutor +// where +// H: HostFunctions, +// { +// type Error = Error; + +// fn call( +// &self, +// ext: &mut dyn Externalities, +// runtime_code: &RuntimeCode, +// method: &str, +// data: &[u8], +// context: CallContext, +// ) -> (Result>, bool) { +// tracing::trace!( +// target: "executor", +// %method, +// "Executing function", +// ); + +// let on_chain_heap_alloc_strategy = if self.ignore_onchain_heap_pages { +// self.default_onchain_heap_alloc_strategy +// } else { +// runtime_code +// .heap_pages +// .map(|h| HeapAllocStrategy::Static { extra_pages: h as _ }) +// .unwrap_or_else(|| self.default_onchain_heap_alloc_strategy) +// }; + +// let heap_alloc_strategy = match context { +// CallContext::Offchain => self.default_offchain_heap_alloc_strategy, +// CallContext::Onchain => on_chain_heap_alloc_strategy, +// }; + +// let result = self.with_instance( +// runtime_code, +// ext, +// heap_alloc_strategy, +// |_, mut instance, _on_chain_version, mut ext| { +// with_externalities_safe(&mut **ext, move || instance.call_export(method, data)) +// }, +// ); + +// (result, false) +// } +// } +func (e WasmExecutor) Call( + ext externalities.Externalities, + runtimeCode core.RuntimeCode, + method string, + data []byte, + context core.CallContext, +) (result []byte, native bool, err error) { + panic("unimpl") +} + +// impl RuntimeVersionOf for WasmExecutor +// where +// H: HostFunctions, +// { +// fn runtime_version( +// &self, +// ext: &mut dyn Externalities, +// runtime_code: &RuntimeCode, +// ) -> Result { +// let on_chain_heap_pages = if self.ignore_onchain_heap_pages { +// self.default_onchain_heap_alloc_strategy +// } else { +// runtime_code +// .heap_pages +// .map(|h| HeapAllocStrategy::Static { extra_pages: h as _ }) +// .unwrap_or_else(|| self.default_onchain_heap_alloc_strategy) +// }; + +// self.with_instance( +// runtime_code, +// ext, +// on_chain_heap_pages, +// |_module, _instance, version, _ext| { +// Ok(version.cloned().ok_or_else(|| Error::ApiError("Unknown version".into()))) +// }, +// ) +// } +// } +func (e WasmExecutor) RuntimeVersion( + externalities externalities.Externalities, + runtimeCode core.RuntimeCode, +) (version.RuntimeVersion, error) { + panic("unimpl") +} diff --git a/internal/client/keystore/keystore.go b/internal/client/keystore/keystore.go index 86d3d8b0ce..4bc57db3a0 100644 --- a/internal/client/keystore/keystore.go +++ b/internal/client/keystore/keystore.go @@ -26,6 +26,6 @@ type KeyStore interface { // Checks if the private keys for the given public key and key type combinations exist. // - // Returns true iffall private keys could be found. + // Returns true iff all private keys could be found. HasKeys(publicKeys []PublicKey) bool } diff --git a/internal/client/mocks/api_ext.go b/internal/client/mocks/api_ext.go index 63ebec8abc..5184327e11 100644 --- a/internal/client/mocks/api_ext.go +++ b/internal/client/mocks/api_ext.go @@ -18,20 +18,20 @@ import ( ) // ApiExt is an autogenerated mock type for the ApiExt type -type ApiExt[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { mock.Mock } -type ApiExt_Expecter[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_Expecter[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { mock *mock.Mock } -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) EXPECT() *ApiExt_Expecter[N, E, H, Hasher, Backend, Result] { - return &ApiExt_Expecter[N, E, H, Hasher, Backend, Result]{mock: &_m.Mock} +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) EXPECT() *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]{mock: &_m.Mock} } // APIVersion provides a mock function with given fields: atHash -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) APIVersion(atHash H) (*uint32, error) { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) APIVersion(atHash H) (*uint32, error) { ret := _m.Called(atHash) if len(ret) == 0 { @@ -61,35 +61,35 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) APIVersion(atHash H) (*uint3 } // ApiExt_APIVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'APIVersion' -type ApiExt_APIVersion_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_APIVersion_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // APIVersion is a helper method to define mock.On call // - atHash H -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) APIVersion(atHash interface{}) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("APIVersion", atHash)} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) APIVersion(atHash interface{}) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("APIVersion", atHash)} } -func (_c *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result]) Run(run func(atHash H)) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(atHash H)) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(H)) }) return _c } -func (_c *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 *uint32, _a1 error) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 *uint32, _a1 error) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0, _a1) return _c } -func (_c *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(H) (*uint32, error)) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(H) (*uint32, error)) *ApiExt_APIVersion_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // ExecuteBlock provides a mock function with given fields: runtimeApiAtParam, block -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExecuteBlock(runtimeApiAtParam H, block runtime.Block[N, H, E]) error { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) ExecuteBlock(runtimeApiAtParam H, block runtime.Block[H, N, E, Header]) error { ret := _m.Called(runtimeApiAtParam, block) if len(ret) == 0 { @@ -97,7 +97,7 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExecuteBlock(runtimeApiAtPar } var r0 error - if rf, ok := ret.Get(0).(func(H, runtime.Block[N, H, E]) error); ok { + if rf, ok := ret.Get(0).(func(H, runtime.Block[H, N, E, Header]) error); ok { r0 = rf(runtimeApiAtParam, block) } else { r0 = ret.Error(0) @@ -107,36 +107,36 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExecuteBlock(runtimeApiAtPar } // ApiExt_ExecuteBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ExecuteBlock' -type ApiExt_ExecuteBlock_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_ExecuteBlock_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // ExecuteBlock is a helper method to define mock.On call // - runtimeApiAtParam H -// - block runtime.Block[N,H,E] -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) ExecuteBlock(runtimeApiAtParam interface{}, block interface{}) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("ExecuteBlock", runtimeApiAtParam, block)} +// - block runtime.Block[H,N,E,Header] +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) ExecuteBlock(runtimeApiAtParam interface{}, block interface{}) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("ExecuteBlock", runtimeApiAtParam, block)} } -func (_c *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result]) Run(run func(runtimeApiAtParam H, block runtime.Block[N, H, E])) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(runtimeApiAtParam H, block runtime.Block[H, N, E, Header])) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(H), args[1].(runtime.Block[N, H, E])) + run(args[0].(H), args[1].(runtime.Block[H, N, E, Header])) }) return _c } -func (_c *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 error) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 error) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0) return _c } -func (_c *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(H, runtime.Block[N, H, E]) error) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(H, runtime.Block[H, N, E, Header]) error) *ApiExt_ExecuteBlock_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // ExecuteInTransaction provides a mock function with given fields: call -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExecuteInTransaction(call func(api.ApiExt[N, E, H, Hasher, Backend, Result]) runtime.TransactionOutcome[Result]) Result { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) ExecuteInTransaction(call func(api.ApiExt[N, E, H, Hasher, Backend, Result, Header]) runtime.TransactionOutcome[Result]) Result { ret := _m.Called(call) if len(ret) == 0 { @@ -144,7 +144,7 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExecuteInTransaction(call fu } var r0 Result - if rf, ok := ret.Get(0).(func(func(api.ApiExt[N, E, H, Hasher, Backend, Result]) runtime.TransactionOutcome[Result]) Result); ok { + if rf, ok := ret.Get(0).(func(func(api.ApiExt[N, E, H, Hasher, Backend, Result, Header]) runtime.TransactionOutcome[Result]) Result); ok { r0 = rf(call) } else { if ret.Get(0) != nil { @@ -156,35 +156,35 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExecuteInTransaction(call fu } // ApiExt_ExecuteInTransaction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ExecuteInTransaction' -type ApiExt_ExecuteInTransaction_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_ExecuteInTransaction_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // ExecuteInTransaction is a helper method to define mock.On call -// - call func(api.ApiExt[N,E,H,Hasher,Backend,Result]) runtime.TransactionOutcome[Result] -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) ExecuteInTransaction(call interface{}) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("ExecuteInTransaction", call)} +// - call func(api.ApiExt[N,E,H,Hasher,Backend,Result,Header]) runtime.TransactionOutcome[Result] +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) ExecuteInTransaction(call interface{}) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("ExecuteInTransaction", call)} } -func (_c *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result]) Run(run func(call func(api.ApiExt[N, E, H, Hasher, Backend, Result]) runtime.TransactionOutcome[Result])) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(call func(api.ApiExt[N, E, H, Hasher, Backend, Result, Header]) runtime.TransactionOutcome[Result])) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(func(api.ApiExt[N, E, H, Hasher, Backend, Result]) runtime.TransactionOutcome[Result])) + run(args[0].(func(api.ApiExt[N, E, H, Hasher, Backend, Result, Header]) runtime.TransactionOutcome[Result])) }) return _c } -func (_c *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 Result) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 Result) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0) return _c } -func (_c *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(func(api.ApiExt[N, E, H, Hasher, Backend, Result]) runtime.TransactionOutcome[Result]) Result) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(func(api.ApiExt[N, E, H, Hasher, Backend, Result, Header]) runtime.TransactionOutcome[Result]) Result) *ApiExt_ExecuteInTransaction_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // ExtractProof provides a mock function with no fields -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExtractProof() *trie.StorageProof { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) ExtractProof() *trie.StorageProof { ret := _m.Called() if len(ret) == 0 { @@ -204,34 +204,34 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ExtractProof() *trie.Storage } // ApiExt_ExtractProof_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ExtractProof' -type ApiExt_ExtractProof_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_ExtractProof_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // ExtractProof is a helper method to define mock.On call -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) ExtractProof() *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("ExtractProof")} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) ExtractProof() *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("ExtractProof")} } -func (_c *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result]) Run(run func()) *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func()) *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } -func (_c *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 *trie.StorageProof) *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 *trie.StorageProof) *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0) return _c } -func (_c *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func() *trie.StorageProof) *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func() *trie.StorageProof) *ApiExt_ExtractProof_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // HasAPI provides a mock function with given fields: atHash -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) HasAPI(atHash H) (bool, error) { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) HasAPI(atHash H) (bool, error) { ret := _m.Called(atHash) if len(ret) == 0 { @@ -259,35 +259,35 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) HasAPI(atHash H) (bool, erro } // ApiExt_HasAPI_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HasAPI' -type ApiExt_HasAPI_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_HasAPI_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // HasAPI is a helper method to define mock.On call // - atHash H -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) HasAPI(atHash interface{}) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("HasAPI", atHash)} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) HasAPI(atHash interface{}) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("HasAPI", atHash)} } -func (_c *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result]) Run(run func(atHash H)) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(atHash H)) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(H)) }) return _c } -func (_c *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 bool, _a1 error) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 bool, _a1 error) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0, _a1) return _c } -func (_c *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(H) (bool, error)) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(H) (bool, error)) *ApiExt_HasAPI_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // HasAPIWith provides a mock function with given fields: atHash, pred -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) HasAPIWith(atHash H, pred func(uint32) bool) (bool, error) { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) HasAPIWith(atHash H, pred func(uint32) bool) (bool, error) { ret := _m.Called(atHash, pred) if len(ret) == 0 { @@ -315,36 +315,36 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) HasAPIWith(atHash H, pred fu } // ApiExt_HasAPIWith_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HasAPIWith' -type ApiExt_HasAPIWith_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_HasAPIWith_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // HasAPIWith is a helper method to define mock.On call // - atHash H // - pred func(uint32) bool -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) HasAPIWith(atHash interface{}, pred interface{}) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("HasAPIWith", atHash, pred)} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) HasAPIWith(atHash interface{}, pred interface{}) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("HasAPIWith", atHash, pred)} } -func (_c *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result]) Run(run func(atHash H, pred func(uint32) bool)) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(atHash H, pred func(uint32) bool)) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(H), args[1].(func(uint32) bool)) }) return _c } -func (_c *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 bool, _a1 error) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 bool, _a1 error) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0, _a1) return _c } -func (_c *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(H, func(uint32) bool) (bool, error)) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(H, func(uint32) bool) (bool, error)) *ApiExt_HasAPIWith_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // IntoStorageChanges provides a mock function with given fields: backend, parentHash -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) IntoStorageChanges(backend Backend, parentHash H) (overlayedchanges.StorageChanges[H, Hasher], error) { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) IntoStorageChanges(backend Backend, parentHash H) (overlayedchanges.StorageChanges[H, Hasher], error) { ret := _m.Called(backend, parentHash) if len(ret) == 0 { @@ -372,36 +372,36 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) IntoStorageChanges(backend B } // ApiExt_IntoStorageChanges_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IntoStorageChanges' -type ApiExt_IntoStorageChanges_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_IntoStorageChanges_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // IntoStorageChanges is a helper method to define mock.On call // - backend Backend // - parentHash H -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) IntoStorageChanges(backend interface{}, parentHash interface{}) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("IntoStorageChanges", backend, parentHash)} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) IntoStorageChanges(backend interface{}, parentHash interface{}) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("IntoStorageChanges", backend, parentHash)} } -func (_c *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result]) Run(run func(backend Backend, parentHash H)) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(backend Backend, parentHash H)) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(Backend), args[1].(H)) }) return _c } -func (_c *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 overlayedchanges.StorageChanges[H, Hasher], _a1 error) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 overlayedchanges.StorageChanges[H, Hasher], _a1 error) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0, _a1) return _c } -func (_c *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(Backend, H) (overlayedchanges.StorageChanges[H, Hasher], error)) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(Backend, H) (overlayedchanges.StorageChanges[H, Hasher], error)) *ApiExt_IntoStorageChanges_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // ProofRecorder provides a mock function with no fields -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ProofRecorder() *api.ProofRecorder[H] { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) ProofRecorder() *api.ProofRecorder[H] { ret := _m.Called() if len(ret) == 0 { @@ -421,137 +421,137 @@ func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) ProofRecorder() *api.ProofRe } // ApiExt_ProofRecorder_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ProofRecorder' -type ApiExt_ProofRecorder_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_ProofRecorder_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // ProofRecorder is a helper method to define mock.On call -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) ProofRecorder() *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("ProofRecorder")} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) ProofRecorder() *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("ProofRecorder")} } -func (_c *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result]) Run(run func()) *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func()) *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } -func (_c *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result]) Return(_a0 *api.ProofRecorder[H]) *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header]) Return(_a0 *api.ProofRecorder[H]) *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(_a0) return _c } -func (_c *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func() *api.ProofRecorder[H]) *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func() *api.ProofRecorder[H]) *ApiExt_ProofRecorder_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return(run) return _c } // RecordProof provides a mock function with no fields -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) RecordProof() { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) RecordProof() { _m.Called() } // ApiExt_RecordProof_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordProof' -type ApiExt_RecordProof_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_RecordProof_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // RecordProof is a helper method to define mock.On call -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) RecordProof() *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("RecordProof")} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) RecordProof() *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("RecordProof")} } -func (_c *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result]) Run(run func()) *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func()) *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } -func (_c *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result]) Return() *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header]) Return() *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return() return _c } -func (_c *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func()) *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func()) *ApiExt_RecordProof_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Run(run) return _c } // RegisterExtension provides a mock function with given fields: extension -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) RegisterExtension(extension interface{}) { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) RegisterExtension(extension interface{}) { _m.Called(extension) } // ApiExt_RegisterExtension_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RegisterExtension' -type ApiExt_RegisterExtension_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_RegisterExtension_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // RegisterExtension is a helper method to define mock.On call // - extension interface{} -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) RegisterExtension(extension interface{}) *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("RegisterExtension", extension)} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) RegisterExtension(extension interface{}) *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("RegisterExtension", extension)} } -func (_c *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result]) Run(run func(extension interface{})) *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(extension interface{})) *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(interface{})) }) return _c } -func (_c *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result]) Return() *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header]) Return() *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return() return _c } -func (_c *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(interface{})) *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(interface{})) *ApiExt_RegisterExtension_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Run(run) return _c } // SetCallContext provides a mock function with given fields: callContext -func (_m *ApiExt[N, E, H, Hasher, Backend, Result]) SetCallContext(callContext core.CallContext) { +func (_m *ApiExt[N, E, H, Hasher, Backend, Result, Header]) SetCallContext(callContext core.CallContext) { _m.Called(callContext) } // ApiExt_SetCallContext_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetCallContext' -type ApiExt_SetCallContext_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}] struct { +type ApiExt_SetCallContext_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]] struct { *mock.Call } // SetCallContext is a helper method to define mock.On call // - callContext core.CallContext -func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result]) SetCallContext(callContext interface{}) *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result] { - return &ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result]{Call: _e.mock.On("SetCallContext", callContext)} +func (_e *ApiExt_Expecter[N, E, H, Hasher, Backend, Result, Header]) SetCallContext(callContext interface{}) *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header] { + return &ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header]{Call: _e.mock.On("SetCallContext", callContext)} } -func (_c *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result]) Run(run func(callContext core.CallContext)) *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header]) Run(run func(callContext core.CallContext)) *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Run(func(args mock.Arguments) { run(args[0].(core.CallContext)) }) return _c } -func (_c *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result]) Return() *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header]) Return() *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Call.Return() return _c } -func (_c *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result]) RunAndReturn(run func(core.CallContext)) *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result] { +func (_c *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header]) RunAndReturn(run func(core.CallContext)) *ApiExt_SetCallContext_Call[N, E, H, Hasher, Backend, Result, Header] { _c.Run(run) return _c } // NewApiExt creates a new instance of ApiExt. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewApiExt[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}](t interface { +func NewApiExt[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, Header runtime.Header[N, H]](t interface { mock.TestingT Cleanup(func()) -}) *ApiExt[N, E, H, Hasher, Backend, Result] { - mock := &ApiExt[N, E, H, Hasher, Backend, Result]{} +}) *ApiExt[N, E, H, Hasher, Backend, Result, Header] { + mock := &ApiExt[N, E, H, Hasher, Backend, Result, Header]{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/internal/client/mocks/backend.go b/internal/client/mocks/backend.go index 91b0fca092..ee3779497f 100644 --- a/internal/client/mocks/backend.go +++ b/internal/client/mocks/backend.go @@ -6,6 +6,8 @@ import ( api "github.com/ChainSafe/gossamer/internal/client/api" blockchain "github.com/ChainSafe/gossamer/internal/primitives/blockchain" + kv "github.com/ChainSafe/gossamer/internal/primitives/kv" + mock "github.com/stretchr/testify/mock" offchain "github.com/ChainSafe/gossamer/internal/primitives/core/offchain" @@ -474,7 +476,7 @@ func (_c *Backend_HaveStateAt_Call[H, N, Hasher, Header, E]) RunAndReturn(run fu } // InsertAux provides a mock function with given fields: insert, delete -func (_m *Backend[H, N, Hasher, Header, E]) InsertAux(insert []api.KeyValue, delete [][]byte) error { +func (_m *Backend[H, N, Hasher, Header, E]) InsertAux(insert []kv.KeyValue, delete [][]byte) error { ret := _m.Called(insert, delete) if len(ret) == 0 { @@ -482,7 +484,7 @@ func (_m *Backend[H, N, Hasher, Header, E]) InsertAux(insert []api.KeyValue, del } var r0 error - if rf, ok := ret.Get(0).(func([]api.KeyValue, [][]byte) error); ok { + if rf, ok := ret.Get(0).(func([]kv.KeyValue, [][]byte) error); ok { r0 = rf(insert, delete) } else { r0 = ret.Error(0) @@ -497,15 +499,15 @@ type Backend_InsertAux_Call[H runtime.Hash, N runtime.Number, Hasher runtime.Has } // InsertAux is a helper method to define mock.On call -// - insert []api.KeyValue +// - insert []kv.KeyValue // - delete [][]byte func (_e *Backend_Expecter[H, N, Hasher, Header, E]) InsertAux(insert interface{}, delete interface{}) *Backend_InsertAux_Call[H, N, Hasher, Header, E] { return &Backend_InsertAux_Call[H, N, Hasher, Header, E]{Call: _e.mock.On("InsertAux", insert, delete)} } -func (_c *Backend_InsertAux_Call[H, N, Hasher, Header, E]) Run(run func(insert []api.KeyValue, delete [][]byte)) *Backend_InsertAux_Call[H, N, Hasher, Header, E] { +func (_c *Backend_InsertAux_Call[H, N, Hasher, Header, E]) Run(run func(insert []kv.KeyValue, delete [][]byte)) *Backend_InsertAux_Call[H, N, Hasher, Header, E] { _c.Call.Run(func(args mock.Arguments) { - run(args[0].([]api.KeyValue), args[1].([][]byte)) + run(args[0].([]kv.KeyValue), args[1].([][]byte)) }) return _c } @@ -515,7 +517,7 @@ func (_c *Backend_InsertAux_Call[H, N, Hasher, Header, E]) Return(_a0 error) *Ba return _c } -func (_c *Backend_InsertAux_Call[H, N, Hasher, Header, E]) RunAndReturn(run func([]api.KeyValue, [][]byte) error) *Backend_InsertAux_Call[H, N, Hasher, Header, E] { +func (_c *Backend_InsertAux_Call[H, N, Hasher, Header, E]) RunAndReturn(run func([]kv.KeyValue, [][]byte) error) *Backend_InsertAux_Call[H, N, Hasher, Header, E] { _c.Call.Return(run) return _c } diff --git a/internal/client/mocks/construct_runtime_api.go b/internal/client/mocks/construct_runtime_api.go index 83380450ba..b614f15d52 100644 --- a/internal/client/mocks/construct_runtime_api.go +++ b/internal/client/mocks/construct_runtime_api.go @@ -2,82 +2,75 @@ package mocks -import ( - api "github.com/ChainSafe/gossamer/internal/primitives/api" - mock "github.com/stretchr/testify/mock" - - runtime "github.com/ChainSafe/gossamer/internal/primitives/runtime" - - statemachine "github.com/ChainSafe/gossamer/internal/primitives/state-machine" -) +import mock "github.com/stretchr/testify/mock" // ConstructRuntimeApi is an autogenerated mock type for the ConstructRuntimeApi type -type ConstructRuntimeApi[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, RuntimeApi api.ApiExt[N, E, H, Hasher, Backend, Result]] struct { +type ConstructRuntimeApi[RuntimeAPI interface{}] struct { mock.Mock } -type ConstructRuntimeApi_Expecter[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, RuntimeApi api.ApiExt[N, E, H, Hasher, Backend, Result]] struct { +type ConstructRuntimeApi_Expecter[RuntimeAPI interface{}] struct { mock *mock.Mock } -func (_m *ConstructRuntimeApi[N, E, H, Hasher, Backend, Result, RuntimeApi]) EXPECT() *ConstructRuntimeApi_Expecter[N, E, H, Hasher, Backend, Result, RuntimeApi] { - return &ConstructRuntimeApi_Expecter[N, E, H, Hasher, Backend, Result, RuntimeApi]{mock: &_m.Mock} +func (_m *ConstructRuntimeApi[RuntimeAPI]) EXPECT() *ConstructRuntimeApi_Expecter[RuntimeAPI] { + return &ConstructRuntimeApi_Expecter[RuntimeAPI]{mock: &_m.Mock} } -// ConstructRuntimeApi provides a mock function with no fields -func (_m *ConstructRuntimeApi[N, E, H, Hasher, Backend, Result, RuntimeApi]) ConstructRuntimeApi() RuntimeApi { +// ConstructRuntimeAPI provides a mock function with no fields +func (_m *ConstructRuntimeApi[RuntimeAPI]) ConstructRuntimeAPI() RuntimeAPI { ret := _m.Called() if len(ret) == 0 { - panic("no return value specified for ConstructRuntimeApi") + panic("no return value specified for ConstructRuntimeAPI") } - var r0 RuntimeApi - if rf, ok := ret.Get(0).(func() RuntimeApi); ok { + var r0 RuntimeAPI + if rf, ok := ret.Get(0).(func() RuntimeAPI); ok { r0 = rf() } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(RuntimeApi) + r0 = ret.Get(0).(RuntimeAPI) } } return r0 } -// ConstructRuntimeApi_ConstructRuntimeApi_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConstructRuntimeApi' -type ConstructRuntimeApi_ConstructRuntimeApi_Call[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, RuntimeApi api.ApiExt[N, E, H, Hasher, Backend, Result]] struct { +// ConstructRuntimeApi_ConstructRuntimeAPI_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ConstructRuntimeAPI' +type ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI interface{}] struct { *mock.Call } -// ConstructRuntimeApi is a helper method to define mock.On call -func (_e *ConstructRuntimeApi_Expecter[N, E, H, Hasher, Backend, Result, RuntimeApi]) ConstructRuntimeApi() *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi] { - return &ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi]{Call: _e.mock.On("ConstructRuntimeApi")} +// ConstructRuntimeAPI is a helper method to define mock.On call +func (_e *ConstructRuntimeApi_Expecter[RuntimeAPI]) ConstructRuntimeAPI() *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI] { + return &ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI]{Call: _e.mock.On("ConstructRuntimeAPI")} } -func (_c *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi]) Run(run func()) *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi] { +func (_c *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI]) Run(run func()) *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI] { _c.Call.Run(func(args mock.Arguments) { run() }) return _c } -func (_c *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi]) Return(_a0 RuntimeApi) *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi] { +func (_c *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI]) Return(_a0 RuntimeAPI) *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI] { _c.Call.Return(_a0) return _c } -func (_c *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi]) RunAndReturn(run func() RuntimeApi) *ConstructRuntimeApi_ConstructRuntimeApi_Call[N, E, H, Hasher, Backend, Result, RuntimeApi] { +func (_c *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI]) RunAndReturn(run func() RuntimeAPI) *ConstructRuntimeApi_ConstructRuntimeAPI_Call[RuntimeAPI] { _c.Call.Return(run) return _c } // NewConstructRuntimeApi creates a new instance of ConstructRuntimeApi. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewConstructRuntimeApi[N runtime.Number, E runtime.Extrinsic, H runtime.Hash, Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result interface{}, RuntimeApi api.ApiExt[N, E, H, Hasher, Backend, Result]](t interface { +func NewConstructRuntimeApi[RuntimeAPI interface{}](t interface { mock.TestingT Cleanup(func()) -}) *ConstructRuntimeApi[N, E, H, Hasher, Backend, Result, RuntimeApi] { - mock := &ConstructRuntimeApi[N, E, H, Hasher, Backend, Result, RuntimeApi]{} +}) *ConstructRuntimeApi[RuntimeAPI] { + mock := &ConstructRuntimeApi[RuntimeAPI]{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/internal/client/network/common/sync/sync.go b/internal/client/network/common/sync/sync.go new file mode 100644 index 0000000000..9ccaf0fdf6 --- /dev/null +++ b/internal/client/network/common/sync/sync.go @@ -0,0 +1,49 @@ +package sync + +// / Sync operation mode. +type SyncMode interface { + /// Returns `true` if `self` is [`Self::Warp`]. + IsWarp() bool + /// Returns `true` if `self` is [`Self::LightState`]. + LightState() bool +} + +type ( + // / Full block download and verification. + SyncModeFull struct{} + // / Download blocks and the latest state. + SyncModeLightState struct { + /// Skip state proof download and verification. + SkipProofs bool + /// Download indexed transactions for recent blocks. + StorageChainMode bool + } + // / Warp sync - verify authority set transitions and the latest state. + SyncModeWarp struct{} +) + +func (smf SyncModeFull) IsWarp() bool { + return false +} +func (smls SyncModeLightState) IsWarp() bool { + return false +} +func (smw SyncModeWarp) IsWarp() bool { + return true +} + +func (smf SyncModeFull) LightState() bool { + return false +} +func (smls SyncModeLightState) LightState() bool { + return true +} +func (smw SyncModeWarp) LightState() bool { + return false +} + +// impl Default for SyncMode { +// fn default() -> Self { +// Self::Full +// } +// } diff --git a/internal/client/network/test/test.go b/internal/client/network/test/test.go new file mode 100644 index 0000000000..e9187e8e0a --- /dev/null +++ b/internal/client/network/test/test.go @@ -0,0 +1,84 @@ +package test + +import ( + "github.com/ChainSafe/gossamer/internal/client" + "github.com/ChainSafe/gossamer/internal/client/network/common/sync" + "github.com/ChainSafe/gossamer/internal/primitives/storage" + "github.com/ChainSafe/gossamer/internal/test-utils/runtime" + testruntimeclient "github.com/ChainSafe/gossamer/internal/test-utils/runtime/client" +) + +// pub type PeersFullClient = Client< +// +// substrate_test_runtime_client::Backend, +// substrate_test_runtime_client::ExecutorDispatch, +// Block, +// substrate_test_runtime_client::runtime::RuntimeApi, +// +// >; +type PeersFullClient = client.Client[ + runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, runtime.Header, +] + +// #[derive(Clone)] +// pub struct PeersClient { +type PeersClient struct { + // client: Arc, + client *PeersFullClient + // backend: Arc, + backend *testruntimeclient.Backend +} + +// pub struct Peer { +type Peer[D any, BlockImport any] struct { + // pub data: D, + Data D + // client: PeersClient, + client PeersClient + /// We keep a copy of the verifier so that we can invoke it for locally-generated blocks, + /// instead of going through the import queue. + // verifier: VerifierAdapter, + // /// We keep a copy of the block_import so that we can invoke it for locally-generated blocks, + // /// instead of going through the import queue. + // block_import: BlockImportAdapter, + // select_chain: Option>, + // backend: Option>, + // network: NetworkWorker::Hash>, + // sync_service: Arc>, + // imported_blocks_stream: Pin> + Send>>, + // finality_notification_stream: Pin> + Send>>, + // listen_addr: Multiaddr, + // notification_services: HashMap>, +} + +// / Configuration for a full peer. +// #[derive(Default)] +// pub struct FullPeerConfig { +type FullPeerConfig struct { + /// Pruning window size. + /// + /// NOTE: only finalized blocks are subject for removal! + BlocksPruning *uint32 + // /// Block announce validator. + // pub block_announce_validator: Option + Send + Sync>>, + // /// List of notification protocols that the network must support. + // pub notifications_protocols: Vec, + // /// List of request-response protocols that the network must support. + // pub request_response_protocols: Vec, + // /// The indices of the peers the peer should be connected to. + // /// + // /// If `None`, it will be connected to all other peers. + // pub connect_to_peers: Option>, + // /// Whether the full peer should have the authority role. + // pub is_authority: bool, + /// Syncing mode + SyncMode sync.SyncMode + /// Extra genesis storage. + ExtraStorage *storage.Storage + /// Enable transaction indexing. + StorageChain bool + // /// Optional target block header to sync to + // pub target_header: Option<::Header>, + /// Force genesis even in case of warp & light state sync. + ForceGenesis bool +} diff --git a/internal/client/utils/notification/notification.go b/internal/client/utils/notification/notification.go index 9f0bd446c8..75cca6f965 100644 --- a/internal/client/utils/notification/notification.go +++ b/internal/client/utils/notification/notification.go @@ -5,12 +5,40 @@ package notification import "github.com/ChainSafe/gossamer/internal/client/utils/pubsub" +// / The receiving half of the notifications channel. +// / +// / The [`NotificationStream`] entity stores the [`Hub`] so it can be +// / used to add more subscriptions. +// #[derive(Clone)] +// pub struct NotificationStream { +type NotificationStream[Payload any] struct { + // hub: Hub, + hub *pubsub.Hub[struct{}, func() (Payload, error), Payload, *registry[Payload]] + // _pd: std::marker::PhantomData, +} + +// impl NotificationStream { +// /// Creates a new pair of receiver and sender of `Payload` notifications. +// pub fn channel() -> (NotificationSender, Self) { +// let hub = Hub::new(TK::TRACING_KEY); +// let sender = NotificationSender { hub: hub.clone() }; +// let receiver = NotificationStream { hub, _pd: Default::default() }; +// (sender, receiver) +// } + +// /// Subscribe to a channel through which the generic payload can be received. +// pub fn subscribe(&self, queue_size_warning: usize) -> NotificationReceiver { +// let receiver = self.hub.subscribe((), queue_size_warning); +// NotificationReceiver { receiver } +// } +// } + // NotificationSender is the sending half of the notifications channel(s). type NotificationSender[Payload any] struct { - pubsub.Hub[struct{}, func() (Payload, error), Payload, *registry[Payload]] + hub *pubsub.Hub[struct{}, func() (Payload, error), Payload, *registry[Payload]] } // Notify sends out a notification to all subscribers that a new payload is available for a block. func (ns *NotificationSender[Payload]) Notify(makePayload func() (Payload, error)) error { - return ns.Hub.Send(makePayload) + return ns.hub.Send(makePayload) } diff --git a/internal/hash-db/hash_db.go b/internal/hash-db/hash_db.go index aea78cf72c..0dde71e267 100644 --- a/internal/hash-db/hash_db.go +++ b/internal/hash-db/hash_db.go @@ -21,10 +21,18 @@ type Prefix struct { // Can be use when the prefix is not used internally or for root nodes. var EmptyPrefix = Prefix{} +type Hash interface { + constraints.Ordered + // Bytes returns a byte slice representation of Hash + Bytes() []byte + // Length return the byte length of the hash + Length() int +} + // Hasher is an interface describing an object that can hash a slice of bytes. Used to abstract // other types over the hashing algorithm. Defines a single hash method and an // Out associated type with the necessary bounds. -type Hasher[Out constraints.Ordered] interface { +type Hasher[Out Hash] interface { // Compute the hash of the provided slice of bytes returning the Out type of the Hasher. Hash(x []byte) Out } diff --git a/internal/memory-db/memory_db.go b/internal/memory-db/memory_db.go index b43a631a4c..68efa36228 100644 --- a/internal/memory-db/memory_db.go +++ b/internal/memory-db/memory_db.go @@ -15,11 +15,7 @@ type dataRC struct { RC int32 } -type Hash interface { - constraints.Ordered - Bytes() []byte -} - +type Hash = hashdb.Hash type Value interface { ~[]byte } diff --git a/internal/memory-db/memory_db_test.go b/internal/memory-db/memory_db_test.go index b0100cc4d4..f77413210f 100644 --- a/internal/memory-db/memory_db_test.go +++ b/internal/memory-db/memory_db_test.go @@ -8,7 +8,7 @@ import ( hashdb "github.com/ChainSafe/gossamer/internal/hash-db" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/core/hashing" + "github.com/ChainSafe/gossamer/internal/primitives/crypto/hashing" "github.com/stretchr/testify/assert" ) diff --git a/internal/primitives/api/api.go b/internal/primitives/api/api.go index 774c342e90..0d89fae9c4 100644 --- a/internal/primitives/api/api.go +++ b/internal/primitives/api/api.go @@ -10,22 +10,22 @@ import ( "github.com/ChainSafe/gossamer/internal/primitives/state-machine/overlayedchanges" ptrie "github.com/ChainSafe/gossamer/internal/primitives/trie" "github.com/ChainSafe/gossamer/internal/primitives/trie/recorder" + "github.com/ChainSafe/gossamer/internal/primitives/version" ) // Something that provides a runtime api. -type ProvideRuntimeApi[Api any] interface { +type ProvideRuntimeAPI[API any] interface { // Returns the runtime api. - // The returned instance will keep track of modifications to the storage. Any successful - // call to an api function, will `commit` its changes to an internal buffer. Otherwise, - // the modifications will be `discarded`. The modifications will not be applied to the - // storage, even on a `commit`. - RuntimeApi() Api + // The returned instance will keep track of modifications to the storage. Any successful call to an api function, + // will commit its changes to an internal buffer. Otherwise, the modifications will be discarded. The modifications + // will not be applied to the storage, even on a commit. + RuntimeAPI() API } // Something that can be constructed to a runtime api. -type ConstructRuntimeApi[RuntimeApi any] interface { +type ConstructRuntimeApi[RuntimeAPI any] interface { // Construct an instance of the runtime api. - ConstructRuntimeApi() RuntimeApi + ConstructRuntimeAPI() RuntimeAPI } // A type that records all accessed trie nodes and generates a proof out of it. @@ -39,11 +39,12 @@ type ApiExt[ Hasher runtime.Hasher[H], Backend statemachine.Backend[H, Hasher], Result any, + Header runtime.Header[N, H], ] interface { // Execute the given closure inside a new transaction. // Depending on the outcome of the closure, the transaction is committed or rolled-back. // The internal result of the closure is returned afterwards. - ExecuteInTransaction(call func(api ApiExt[N, E, H, Hasher, Backend, Result]) runtime.TransactionOutcome[Result]) Result + ExecuteInTransaction(call func(api ApiExt[N, E, H, Hasher, Backend, Result, Header]) runtime.TransactionOutcome[Result]) Result // Checks if the given api is implemented and versions match. HasAPI(atHash H) (bool, error) // Check if the given api is implemented and the version passes a predicate. @@ -67,5 +68,20 @@ type ApiExt[ // Register an [Extension] that will be accessible while executing a runtime api call. RegisterExtension(extension any) // Execute the given block - ExecuteBlock(runtimeApiAtParam H, block runtime.Block[N, H, E]) error + ExecuteBlock(runtimeApiAtParam H, block runtime.Block[H, N, E, Header]) error +} + +// pub trait Core { +type Core[H runtime.Hash] interface { + /// Returns the version of the runtime. + // fn version() -> RuntimeVersion; + Version(hash H) (version.RuntimeVersion, error) + // /// Execute the given block. + // fn execute_block(block: Block); + // /// Initialize a block with the given header. + // #[changed_in(5)] + // #[renamed("initialise_block", 2)] + // fn initialize_block(header: &::Header); + // /// Initialize a block with the given header and return the runtime executive mode. + // fn initialize_block(header: &::Header) -> ExtrinsicInclusionMode; } diff --git a/internal/primitives/blockchain/backend.go b/internal/primitives/blockchain/backend.go index 192d7a136a..8f1c89981d 100644 --- a/internal/primitives/blockchain/backend.go +++ b/internal/primitives/blockchain/backend.go @@ -107,7 +107,7 @@ type BlockBackend[ BlockIndexedBody(hash H) ([][]byte, error) // Block gets the full block by hash. - Block(hash H) (*generic.SignedBlock[N, H, Hasher, E], error) + Block(hash H) (*generic.SignedBlock[N, H, Hasher, E, Header], error) // BlockStatus gets block status by block hash. BlockStatus(hash H) (common.BlockStatus, error) diff --git a/internal/primitives/blockchain/error.go b/internal/primitives/blockchain/error.go index d2c6acef9e..bd171afabb 100644 --- a/internal/primitives/blockchain/error.go +++ b/internal/primitives/blockchain/error.go @@ -13,9 +13,12 @@ var ( ErrInvalidState = errors.New("provided state is invalid") ErrInvalidChildStorageKey = errors.New("invalid child storage key") ErrBadJustification = errors.New("bad justification for header") + ErrMissingHeader = errors.New("Failed to get header for hash") ErrStateDatabase = errors.New("state database error") ErrSetHeadTooOld = errors.New("failed to set the chain head to a block that's too old") ErrInvalidStateRoot = errors.New("calculated state root does not match") ErrIncompletePipeline = errors.New("incomplete block import pipeline") ErrVersionInvalid = errors.New("failed to get runtime version") + ErrApplication = errors.New("application error") + ErrStorage = errors.New("storage error") ) diff --git a/internal/client/consensus/common/select_chain.go b/internal/primitives/consensus/common/select_chain.go similarity index 85% rename from internal/client/consensus/common/select_chain.go rename to internal/primitives/consensus/common/select_chain.go index 99a4eda353..569d1ecaaa 100644 --- a/internal/client/consensus/common/select_chain.go +++ b/internal/primitives/consensus/common/select_chain.go @@ -5,18 +5,15 @@ package common import "github.com/ChainSafe/gossamer/internal/primitives/runtime" -// The SelectChain interface defines the strategy upon which the head is chosen if multiple forks are present for an -// opaque definition of "best" in the specific chain build. +// The SelectChain interface defines the strategy upon which the head is chosen if multiple forks are present for an opaque +// definition of "best" in the specific chain build. // // The Strategy can be customised for the two use cases of authoring new blocks upon the best chain or which fork to // finalize. type SelectChain[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]] interface { // Get all leaves of the chain, i.e. block hashes that have no children currently. // Leaves that can never be finalized will not be returned. - Leaves() <-chan struct { - Leaves []H - Error error - } + Leaves() <-chan LeavesError[H] // Among those leaves deterministically pick one chain as the generally best chain to author new blocks upon and // probably (but not necessarily) finalize. @@ -27,6 +24,11 @@ type SelectChain[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]] FinalityTarget(baseHash H, maybeMaxNumber *N) <-chan HashError[H] } +type LeavesError[H runtime.Hash] struct { + Leaves []H + Error error +} + type HeaderError[H runtime.Hash, N runtime.Number, Header runtime.Header[N, H]] struct { Header Header Error error diff --git a/internal/primitives/consensus/grandpa/grandpa.go b/internal/primitives/consensus/grandpa/grandpa.go index b6c5426ca9..7618a21e77 100644 --- a/internal/primitives/consensus/grandpa/grandpa.go +++ b/internal/primitives/consensus/grandpa/grandpa.go @@ -4,8 +4,11 @@ package grandpa import ( + "fmt" + "github.com/ChainSafe/gossamer/internal/client/keystore" "github.com/ChainSafe/gossamer/internal/log" + "github.com/ChainSafe/gossamer/internal/primitives/api" "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa/app" "github.com/ChainSafe/gossamer/internal/primitives/core/crypto" "github.com/ChainSafe/gossamer/internal/primitives/runtime" @@ -96,7 +99,145 @@ type GrandpaJustification[H runtime.Hash, N runtime.Number, Header runtime.Heade VoteAncestries []Header } -// EquivocationProof is proof of voter misbehavior on a given set id. Misbehavior/equivocation in GRANDPA happens when +// / An consensus log item for GRANDPA. +// #[derive(Decode, Encode, PartialEq, Eq, Clone, RuntimeDebug)] +// #[cfg_attr(feature = "serde", derive(Serialize))] +// pub enum ConsensusLog { +type ConsensusLog interface { + isConsensusLog() +} + +// / Schedule an authority set change. +// / +// / The earliest digest of this type in a single block will be respected, +// / provided that there is no `ForcedChange` digest. If there is, then the +// / `ForcedChange` will take precedence. +// / +// / No change should be scheduled if one is already and the delay has not +// / passed completely. +// / +// / This should be a pure function: i.e. as long as the runtime can interpret +// / the digest type it should return the same result regardless of the current +// / state. +// #[codec(index = 1)] +// ScheduledChange(ScheduledChange), +type ConensusLogScheduledChange[N runtime.Number] ScheduledChange[N] + +// / Force an authority set change. +// / +// / Forced changes are applied after a delay of _imported_ blocks, +// / while pending changes are applied after a delay of _finalized_ blocks. +// / +// / The earliest digest of this type in a single block will be respected, +// / with others ignored. +// / +// / No change should be scheduled if one is already and the delay has not +// / passed completely. +// / +// / This should be a pure function: i.e. as long as the runtime can interpret +// / the digest type it should return the same result regardless of the current +// / state. +// +// #[codec(index = 2)] +// ForcedChange(N, ScheduledChange), +type ConsensusLogForcedChange[N runtime.Number] struct { + Median N + ScheduledChange[N] +} + +// / Note that the authority with given index is disabled until the next change. +// +// #[codec(index = 3)] +// OnDisabled(AuthorityIndex), +type ConsensusLogOnDisabled AuthorityIndex + +// / A signal to pause the current authority set after the given delay. +// / After finalizing the block at _delay_ the authorities should stop voting. +// #[codec(index = 4)] +// Pause(N), +type ConsensusLogPause[N runtime.Number] struct { + Delay N +} + +// / A signal to resume the current authority set after the given delay. +// / After authoring the block at _delay_ the authorities should resume voting. +// +// #[codec(index = 5)] +// Resume(N), +type ConsensusLogResume[N runtime.Number] struct { + Delay N +} + +func (ConensusLogScheduledChange[N]) isConsensusLog() {} +func (ConsensusLogForcedChange[N]) isConsensusLog() {} +func (ConsensusLogOnDisabled) isConsensusLog() {} +func (ConsensusLogPause[N]) isConsensusLog() {} +func (ConsensusLogResume[N]) isConsensusLog() {} + +type ConsensusLogVDT[N runtime.Number] struct { + inner ConsensusLog +} + +func (mvdt *ConsensusLogVDT[N]) SetValue(value any) (err error) { + switch value := value.(type) { + case ConensusLogScheduledChange[N]: + mvdt.inner = value + return + case ConsensusLogForcedChange[N]: + mvdt.inner = value + return + case ConsensusLogOnDisabled: + mvdt.inner = value + return + case ConsensusLogPause[N]: + mvdt.inner = value + return + case ConsensusLogResume[N]: + mvdt.inner = value + return + default: + return fmt.Errorf("unsupported type") + } +} + +func (mvdt ConsensusLogVDT[N]) IndexValue() (index uint, value any, err error) { + switch mvdt.inner.(type) { + case ConensusLogScheduledChange[N]: + return 1, mvdt.inner, nil + case ConsensusLogForcedChange[N]: + return 2, mvdt.inner, nil + case ConsensusLogOnDisabled: + return 3, mvdt.inner, nil + case ConsensusLogPause[N]: + return 4, mvdt.inner, nil + case ConsensusLogResume[N]: + return 5, mvdt.inner, nil + } + return 0, nil, scale.ErrUnsupportedVaryingDataTypeValue +} + +func (mvdt ConsensusLogVDT[N]) Value() (value any, err error) { + _, value, err = mvdt.IndexValue() + return +} + +func (mvdt ConsensusLogVDT[N]) ValueAt(index uint) (value any, err error) { + switch index { + case 1: + return ConensusLogScheduledChange[N]{}, nil + case 2: + return ConsensusLogForcedChange[N]{}, nil + case 3: + return ConsensusLogOnDisabled(0), nil + case 4: + return ConsensusLogPause[N]{}, nil + case 5: + return ConsensusLogResume[N]{}, nil + } + return nil, scale.ErrUnknownVaryingDataTypeValue +} + +// EquiovcationProof is proof of voter misbehavior on a given set id. Misbehavior/equivocation in GRANDPA happens when // a voter votes on the same round (either at prevote or precommit stage) for different blocks. Proving is achieved by // collecting the signed messages of conflicting votes. type EquivocationProof[H runtime.Hash, N runtime.Number] struct { @@ -211,12 +352,13 @@ type OpaqueKeyOwnershipProof = runtime.OpaqueValue // // The consensus protocol will coordinate the handoff externally. type GrandpaAPI[H runtime.Hash, N runtime.Number] interface { + api.Core[H] // Get the current GRANDPA authorities and weights. This should not change except for when changes are scheduled // and the corresponding delay has passed. // // When called at block B, it will return the set of authorities that should be used to finalize descendants of // this block (B+1, B+2, ...). The block B itself is finalized by the authorities from block B-1. - GrandpaAuthorities() AuthorityList + GrandpaAuthorities(hash H) (AuthorityList, error) // Submits an unsigned extrinsic to report an equivocation. The caller must provide the equivocation proof and a // key ownership proof (should be obtained using GenerateKeyOwnershipProof). The extrinsic will be unsigned @@ -242,4 +384,8 @@ type GrandpaAPI[H runtime.Hash, N runtime.Number] interface { setID SetID, authorityID AuthorityID, ) *OpaqueKeyOwnershipProof + + /// Get current GRANDPA authority set id. + // fn current_set_id() -> SetId; + CurrentSetID(hash H) (SetID, error) } diff --git a/internal/primitives/core/crypto/crypto.go b/internal/primitives/core/crypto/crypto.go index 4227abfa80..41f1fe6315 100644 --- a/internal/primitives/core/crypto/crypto.go +++ b/internal/primitives/core/crypto/crypto.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/ChainSafe/gossamer/internal/primitives/core/hashing" + "github.com/ChainSafe/gossamer/internal/primitives/crypto/hashing" "github.com/ChainSafe/gossamer/pkg/scale" ) diff --git a/internal/primitives/core/ed25519/ed25519.go b/internal/primitives/core/ed25519/ed25519.go index e5a7e7a78b..acfc799f03 100644 --- a/internal/primitives/core/ed25519/ed25519.go +++ b/internal/primitives/core/ed25519/ed25519.go @@ -14,7 +14,7 @@ import ( "github.com/ChainSafe/go-schnorrkel" "github.com/ChainSafe/gossamer/internal/primitives/core/crypto" - "github.com/ChainSafe/gossamer/internal/primitives/core/hashing" + "github.com/ChainSafe/gossamer/internal/primitives/crypto/hashing" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/tyler-smith/go-bip39" ) diff --git a/internal/primitives/core/hasher/hasher.go b/internal/primitives/core/hasher/hasher.go new file mode 100644 index 0000000000..328aba8a20 --- /dev/null +++ b/internal/primitives/core/hasher/hasher.go @@ -0,0 +1,13 @@ +package hasher + +import ( + "github.com/ChainSafe/gossamer/internal/primitives/core/hash" + "github.com/ChainSafe/gossamer/internal/primitives/crypto/hashing" +) + +type Blake2Hasher struct{} + +func (bt256 Blake2Hasher) Hash(s []byte) hash.H256 { + h := hashing.BlakeTwo256(s) + return hash.H256(h[:]) +} diff --git a/internal/primitives/core/hashing/hashing.go b/internal/primitives/core/hashing/hashing.go deleted file mode 100644 index 086edef4ab..0000000000 --- a/internal/primitives/core/hashing/hashing.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2024 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package hashing - -import ( - "golang.org/x/crypto/blake2b" - "golang.org/x/crypto/sha3" -) - -// BlakeTwo256 returns a Blake2 256-bit hash of the input data -func BlakeTwo256(data []byte) [32]byte { - h, err := blake2b.New256(nil) - if err != nil { - panic(err) - } - _, err = h.Write(data) - if err != nil { - panic(err) - } - encoded := h.Sum(nil) - var arr [32]byte - copy(arr[:], encoded) - return arr -} - -// Keccak256 returns the keccak256 hash of the input data -func Keccak256(data []byte) [32]byte { - h := sha3.NewLegacyKeccak256() - _, err := h.Write(data) - if err != nil { - panic(err) - } - - hash := h.Sum(nil) - var buf = [32]byte{} - copy(buf[:], hash) - return buf -} diff --git a/internal/primitives/core/interfaces.go b/internal/primitives/core/interfaces.go index 4062f353e2..8e2368861f 100644 --- a/internal/primitives/core/interfaces.go +++ b/internal/primitives/core/interfaces.go @@ -5,7 +5,23 @@ package core import "github.com/ChainSafe/gossamer/internal/primitives/externalities" +/// The context in which a call is done. +/// +/// Depending on the context the executor may chooses different kind of heap sizes for the runtime +/// instance. +// #[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd)] +// pub enum CallContext { +// /// The call is happening in some offchain context. +// Offchain, +// /// The call is happening in some on-chain context like building or importing a block. +// Onchain, +// } + +// / Code execution engine. +// pub trait CodeExecutor: Sized + Send + Sync + ReadRuntimeVersion + Clone + 'static { type CodeExecutor interface { + ReadRuntimeVersion + // Call a given method in the runtime. // Returns a tuple of the result (either the output data or an execution error) together with a // bool, which is true if native execution was used. @@ -17,3 +33,34 @@ type CodeExecutor interface { context CallContext, ) (result []byte, native bool, err error) } + +// / A trait that allows reading version information from the binary. +// pub trait ReadRuntimeVersion: Send + Sync { +type ReadRuntimeVersion interface { + // /// Reads the runtime version information from the given wasm code. + // /// + // /// The version information may be embedded into the wasm binary itself. If it is not present, + // /// then this function may fallback to the legacy way of reading the version. + // /// + // /// The legacy mechanism involves instantiating the passed wasm runtime and calling + // /// `Core_version` on it. This is a very expensive operation. + // /// + // /// `ext` is only needed in case the calling into runtime happens. Otherwise it is ignored. + // /// + // /// Compressed wasm blobs are supported and will be decompressed if needed. If uncompression + // /// fails, the error is returned. + // /// + // /// # Errors + // /// + // /// If the version information present in binary, but is corrupted - returns an error. + // /// + // /// Otherwise, if there is no version information present, and calling into the runtime takes + // /// place, then an error would be returned if `Core_version` is not provided. + // fn read_runtime_version( + // + // &self, + // wasm_code: &[u8], + // ext: &mut dyn Externalities, + // + // ) -> Result, String>; +} diff --git a/internal/primitives/core/sr25519/sr25519.go b/internal/primitives/core/sr25519/sr25519.go new file mode 100644 index 0000000000..b636dd5e8a --- /dev/null +++ b/internal/primitives/core/sr25519/sr25519.go @@ -0,0 +1,60 @@ +package sr25519 + +// / An Schnorrkel/Ristretto x25519 ("sr25519") public key. +type Public [32]byte + +// / An Schnorrkel/Ristretto x25519 ("sr25519") signature. +type Signature [64]byte + +// / Transcript ready to be used for VRF related operations. +// #[derive(Clone)] +// pub struct VrfTranscript(pub merlin::Transcript); +type VRFTranscript struct { +} + +// / VRF input ready to be used for VRF sign and verify operations. +type VRFSignData struct { + /// Transcript data contributing to VRF output. + transcript VRFTranscript + /// Extra transcript data to be signed by the VRF. + extra *VRFTranscript +} + +// /// Transcript ready to be used for VRF related operations. +// #[derive(Clone)] +// pub struct VrfTranscript(pub merlin::Transcript); + +// impl VrfTranscript { +// /// Build a new transcript instance. +// /// +// /// Each `data` element is a tuple `(domain, message)` composing the transcipt. +// pub fn new(label: &'static [u8], data: &[(&'static [u8], &[u8])]) -> Self { +// let mut transcript = merlin::Transcript::new(label); +// data.iter().for_each(|(l, b)| transcript.append_message(l, b)); +// VrfTranscript(transcript) +// } + +// /// Map transcript to `VrfSignData`. +// pub fn into_sign_data(self) -> VrfSignData { +// self.into() +// } +// } + +// VRFInput is a transcript used by the Fiat-Shamir transform. +type VRFInput VRFTranscript + +// / VRF output type suitable for schnorrkel operations. +// #[derive(Clone, Debug, PartialEq, Eq)] +// pub struct VrfOutput(pub schnorrkel::vrf::VRFOutput); +type VRFOutput struct{} + +// /// VRF proof type suitable for schnorrkel operations. +// #[derive(Clone, Debug, PartialEq, Eq)] +// pub struct VrfProof(pub schnorrkel::vrf::VRFProof); +type VRFProof struct{} + +// / VRF signature data +type VRFSignature struct { + Output VRFOutput + Proof VRFProof +} diff --git a/internal/primitives/crypto/hashing/hashing.go b/internal/primitives/crypto/hashing/hashing.go new file mode 100644 index 0000000000..47cee391f7 --- /dev/null +++ b/internal/primitives/crypto/hashing/hashing.go @@ -0,0 +1,71 @@ +// Copyright 2024 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package hashing + +import ( + "encoding/binary" + + "github.com/OneOfOne/xxhash" + "golang.org/x/crypto/blake2b" + "golang.org/x/crypto/sha3" +) + +// BlakeTwo256 returns a Blake2 256-bit hash of the input data +func BlakeTwo256(data []byte) [32]byte { + h, err := blake2b.New256(nil) + if err != nil { + panic(err) + } + _, err = h.Write(data) + if err != nil { + panic(err) + } + encoded := h.Sum(nil) + var arr [32]byte + copy(arr[:], encoded) + return arr +} + +// Keccak256 returns the keccak256 hash of the input data +func Keccak256(data []byte) [32]byte { + h := sha3.NewLegacyKeccak256() + _, err := h.Write(data) + if err != nil { + panic(err) + } + + hash := h.Sum(nil) + var buf = [32]byte{} + copy(buf[:], hash) + return buf +} + +// / Do a XX 128-bit hash and return result. +func Twox128(data []byte) [16]byte { + // compute xxHash64 twice with seeds 0 and 1 applied on given byte array + h0 := xxhash.NewS64(0) // create xxHash with 0 seed + _, err := h0.Write(data) + if err != nil { + panic(err) + } + res0 := h0.Sum64() + hash0 := make([]byte, 8) + binary.LittleEndian.PutUint64(hash0, res0) + + h1 := xxhash.NewS64(1) // create xxHash with 1 seed + _, err = h1.Write(data) + if err != nil { + panic(err) + } + res1 := h1.Sum64() + hash1 := make([]byte, 8) + binary.LittleEndian.PutUint64(hash1, res1) + + return [16]byte{ + hash0[0], hash0[1], hash0[2], hash0[3], + hash0[4], hash0[5], hash0[6], hash0[7], + hash1[0], hash1[1], hash1[2], hash1[3], + hash1[4], hash1[5], hash1[6], hash1[7], + } +} diff --git a/internal/primitives/io/trie/trie.go b/internal/primitives/io/trie/trie.go new file mode 100644 index 0000000000..c6a38d0cc7 --- /dev/null +++ b/internal/primitives/io/trie/trie.go @@ -0,0 +1,157 @@ +package trie + +import ( + "github.com/ChainSafe/gossamer/internal/primitives/core/hash" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" + "github.com/ChainSafe/gossamer/internal/primitives/kv" + "github.com/ChainSafe/gossamer/internal/primitives/storage" + "github.com/ChainSafe/gossamer/internal/primitives/trie" +) + +type KeyValue = kv.KeyValue + +/// Interface that provides trie related functionality. +// #[runtime_interface] +// pub trait Trie { +// /// A trie root formed from the iterated items. +// fn blake2_256_root(input: Vec<(Vec, Vec)>) -> H256 { +// LayoutV0::::trie_root(input) +// } + +// /// A trie root formed from the iterated items. +// #[version(2)] +// +// fn blake2_256_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> H256 { +// match version { +// StateVersion::V0 => LayoutV0::::trie_root(input), +// StateVersion::V1 => LayoutV1::::trie_root(input), +// } +// } +func BlakeTwo256Root(input []KeyValue, version storage.StateVersion) hash.H256 { + switch version { + case storage.StateVersionV0: + return trie.LayoutV0[hasher.Blake2Hasher, hash.H256]{}.TrieRoot(input) + case storage.StateVersionV1: + return trie.LayoutV1[hasher.Blake2Hasher, hash.H256]{}.TrieRoot(input) + default: + panic("unreachable") + } + panic("unreachable") +} + +// /// A trie root formed from the enumerated items. +// fn blake2_256_ordered_root(input: Vec>) -> H256 { +// LayoutV0::::ordered_trie_root(input) +// } + +// /// A trie root formed from the enumerated items. +// #[version(2)] +// fn blake2_256_ordered_root(input: Vec>, version: StateVersion) -> H256 { +// match version { +// StateVersion::V0 => LayoutV0::::ordered_trie_root(input), +// StateVersion::V1 => LayoutV1::::ordered_trie_root(input), +// } +// } + +// /// A trie root formed from the iterated items. +// fn keccak_256_root(input: Vec<(Vec, Vec)>) -> H256 { +// LayoutV0::::trie_root(input) +// } + +// /// A trie root formed from the iterated items. +// #[version(2)] +// fn keccak_256_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> H256 { +// match version { +// StateVersion::V0 => LayoutV0::::trie_root(input), +// StateVersion::V1 => LayoutV1::::trie_root(input), +// } +// } + +// /// A trie root formed from the enumerated items. +// fn keccak_256_ordered_root(input: Vec>) -> H256 { +// LayoutV0::::ordered_trie_root(input) +// } + +// /// A trie root formed from the enumerated items. +// #[version(2)] +// fn keccak_256_ordered_root(input: Vec>, version: StateVersion) -> H256 { +// match version { +// StateVersion::V0 => LayoutV0::::ordered_trie_root(input), +// StateVersion::V1 => LayoutV1::::ordered_trie_root(input), +// } +// } + +// /// Verify trie proof +// fn blake2_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { +// sp_trie::verify_trie_proof::, _, _, _>( +// &root, +// proof, +// &[(key, Some(value))], +// ) +// .is_ok() +// } + +// /// Verify trie proof +// #[version(2)] +// fn blake2_256_verify_proof( +// root: H256, +// proof: &[Vec], +// key: &[u8], +// value: &[u8], +// version: StateVersion, +// ) -> bool { +// match version { +// StateVersion::V0 => sp_trie::verify_trie_proof::< +// LayoutV0, +// _, +// _, +// _, +// >(&root, proof, &[(key, Some(value))]) +// .is_ok(), +// StateVersion::V1 => sp_trie::verify_trie_proof::< +// LayoutV1, +// _, +// _, +// _, +// >(&root, proof, &[(key, Some(value))]) +// .is_ok(), +// } +// } + +// /// Verify trie proof +// fn keccak_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { +// sp_trie::verify_trie_proof::, _, _, _>( +// &root, +// proof, +// &[(key, Some(value))], +// ) +// .is_ok() +// } + +// /// Verify trie proof +// #[version(2)] +// fn keccak_256_verify_proof( +// root: H256, +// proof: &[Vec], +// key: &[u8], +// value: &[u8], +// version: StateVersion, +// ) -> bool { +// match version { +// StateVersion::V0 => sp_trie::verify_trie_proof::< +// LayoutV0, +// _, +// _, +// _, +// >(&root, proof, &[(key, Some(value))]) +// .is_ok(), +// StateVersion::V1 => sp_trie::verify_trie_proof::< +// LayoutV1, +// _, +// _, +// _, +// >(&root, proof, &[(key, Some(value))]) +// .is_ok(), +// } +// } +// } diff --git a/internal/primitives/kv/kv.go b/internal/primitives/kv/kv.go new file mode 100644 index 0000000000..46eeb9aa99 --- /dev/null +++ b/internal/primitives/kv/kv.go @@ -0,0 +1,6 @@ +package kv + +type KeyValue struct { + Key []byte + Value []byte +} diff --git a/internal/primitives/runtime/digest.go b/internal/primitives/runtime/digest.go index 6b60bb5670..aabdcaf197 100644 --- a/internal/primitives/runtime/digest.go +++ b/internal/primitives/runtime/digest.go @@ -5,47 +5,43 @@ package runtime import ( "fmt" + "io" "github.com/ChainSafe/gossamer/pkg/scale" ) -// DigestItemTypes is interface constraint of [DigestItem] -type DigestItemTypes interface { - PreRuntime | Consensus | Seal | Other | RuntimeEnvironmentUpdated -} - -// DigestItem is able to encode/decode "system" digest items and +// DigestItemVDT is able to encode/decode "system" digest items and // provide opaque access to other items. -type DigestItem struct { - inner any +type DigestItemVDT struct { + inner DigestItem } -// NewDigestItem is constructor for DigestItem -func NewDigestItem[T DigestItemTypes](value T) DigestItem { - item := DigestItem{} +// NewDigestItemVDT is constructor for DigestItem +func NewDigestItemVDT(value DigestItem) DigestItemVDT { + item := DigestItemVDT{} setDigestItem(&item, value) return item } -func setDigestItem[Value DigestItemTypes](mvdt *DigestItem, value Value) { +func setDigestItem(mvdt *DigestItemVDT, value DigestItem) { mvdt.inner = value } -func (mvdt *DigestItem) SetValue(value any) (err error) { +func (mvdt *DigestItemVDT) SetValue(value any) (err error) { switch value := value.(type) { - case PreRuntime: + case DigestItemPreRuntime: setDigestItem(mvdt, value) return - case Consensus: + case DigestItemConsensus: setDigestItem(mvdt, value) return - case Seal: + case DigestItemSeal: setDigestItem(mvdt, value) return - case RuntimeEnvironmentUpdated: + case DigestItemRuntimeEnvironmentUpdated: setDigestItem(mvdt, value) return - case Other: + case DigestItemOther: setDigestItem(mvdt, value) return default: @@ -53,48 +49,67 @@ func (mvdt *DigestItem) SetValue(value any) (err error) { } } -func (mvdt DigestItem) IndexValue() (index uint, value any, err error) { +func (mvdt DigestItemVDT) IndexValue() (index uint, value any, err error) { switch mvdt.inner.(type) { - case Other: + case DigestItemOther: return 0, mvdt.inner, nil - case Consensus: + case DigestItemConsensus: return 4, mvdt.inner, nil - case Seal: + case DigestItemSeal: return 5, mvdt.inner, nil - case PreRuntime: + case DigestItemPreRuntime: return 6, mvdt.inner, nil - case RuntimeEnvironmentUpdated: + case DigestItemRuntimeEnvironmentUpdated: return 8, mvdt.inner, nil } return 0, nil, scale.ErrUnsupportedVaryingDataTypeValue } -func (mvdt DigestItem) Value() (value any, err error) { +func (mvdt DigestItemVDT) Value() (value any, err error) { _, value, err = mvdt.IndexValue() return } -func (mvdt DigestItem) ValueAt(index uint) (value any, err error) { +func (mvdt DigestItemVDT) ValueAt(index uint) (value any, err error) { switch index { case 0: - return Other{}, nil + return DigestItemOther{}, nil case 4: - return Consensus{}, nil + return DigestItemConsensus{}, nil case 5: - return Seal{}, nil + return DigestItemSeal{}, nil case 6: - return PreRuntime{}, nil + return DigestItemPreRuntime{}, nil case 8: - return RuntimeEnvironmentUpdated{}, nil + return DigestItemRuntimeEnvironmentUpdated{}, nil } return nil, scale.ErrUnknownVaryingDataTypeValue } -func (mvdt DigestItem) String() string { +func (mvdt DigestItemVDT) String() string { return fmt.Sprintf("%s", mvdt.inner) } -// PreRuntime is a pre-runtime digest. +// DigestItem is able to encode/decode "system" digest items and +// provide opaque access to other items. +type DigestItem interface { + TryAsRaw(id OpaqueDigestItemID) []byte +} + +// DigestItemTryTo tries to match this digest item to the given opaque item identifier; if it matches, then +// try to cast to the given data type; if that works, return it. +func DigestItemTryTo[T any](item DigestItem, id OpaqueDigestItemID) *T { + if raw := item.TryAsRaw(id); raw != nil { + var t T + if err := scale.Unmarshal(raw, &t); err != nil { + return nil + } + return &t + } + return nil +} + +// DigestItemPreRuntime is a pre-runtime digest. // // These are messages from the consensus engine to the runtime, although // the consensus engine can (and should) read them itself to avoid @@ -102,36 +117,74 @@ func (mvdt DigestItem) String() string { // these, but this is not (yet) checked. // // NOTE: the runtime is not allowed to panic or fail in an on_initialize -// call if an expected PreRuntime digest is not present. It is the +// call if an expected DigestItemPreRuntime digest is not present. It is the // responsibility of a external block verifier to check this. Runtime API calls // will initialize the block without pre-runtime digests, so initialization // cannot fail when they are missing. -type PreRuntime struct { +type DigestItemPreRuntime struct { ConsensusEngineID Bytes []byte } -// Consensus is a message from the runtime to the consensus engine. This should *never* +func (pr DigestItemPreRuntime) TryAsRaw(id OpaqueDigestItemID) []byte { + if id, ok := id.(OpaqueDigestItemIDPreRuntime); ok { + if pr.ConsensusEngineID == ConsensusEngineID(id) { + return pr.Bytes + } + } + return nil +} + +// DigestItemConsensus is a message from the runtime to the consensus engine. This should *never* // be generated by the native code of any consensus engine, but this is not // checked (yet). -type Consensus struct { +type DigestItemConsensus struct { ConsensusEngineID Bytes []byte } -// Put a Seal on it. This is only used by native code, and is never seen +func (pr DigestItemConsensus) TryAsRaw(id OpaqueDigestItemID) []byte { + if id, ok := id.(OpaqueDigestItemIDConsensus); ok { + if pr.ConsensusEngineID == ConsensusEngineID(id) { + return pr.Bytes + } + } + return nil +} + +// Put a DigestItemSeal on it. This is only used by native code, and is never seen // by runtimes. -type Seal struct { +type DigestItemSeal struct { ConsensusEngineID Bytes []byte } -// Some Other thing. Unsupported and experimental. -type Other []byte +func (pr DigestItemSeal) TryAsRaw(id OpaqueDigestItemID) []byte { + if id, ok := id.(OpaqueDigestItemIDSeal); ok { + if pr.ConsensusEngineID == ConsensusEngineID(id) { + return pr.Bytes + } + } + return nil +} + +// Some DigestItemOther thing. Unsupported and experimental. +type DigestItemOther []byte -// RuntimeEnvironmentUpdated is an indication for the light clients that the runtime execution +func (pr DigestItemOther) TryAsRaw(id OpaqueDigestItemID) []byte { + if _, ok := id.(OpaqueDigestItemIDOther); ok { + return pr + } + return nil +} + +// DigestItemRuntimeEnvironmentUpdated is an indication for the light clients that the runtime execution // environment is updated. -type RuntimeEnvironmentUpdated struct{} +type DigestItemRuntimeEnvironmentUpdated struct{} + +func (DigestItemRuntimeEnvironmentUpdated) TryAsRaw(id OpaqueDigestItemID) []byte { + return nil +} // Digest is a header digest. type Digest struct { @@ -143,3 +196,56 @@ type Digest struct { func (d *Digest) Push(item DigestItem) { d.Logs = append(d.Logs, item) } + +func (d Digest) MarshalSCALE() ([]byte, error) { + type helper struct { + Logs []DigestItemVDT + } + h := helper{ + Logs: make([]DigestItemVDT, len(d.Logs)), + } + for i, item := range d.Logs { + h.Logs[i] = NewDigestItemVDT(item) + } + return scale.Marshal(h.Logs) +} + +func (d *Digest) UnmarshalSCALE(reader io.Reader) error { + type helper struct { + Logs []DigestItemVDT + } + h := helper{} + decoder := scale.NewDecoder(reader) + err := decoder.Decode(&h) + if err != nil { + return err + } + d.Logs = make([]DigestItem, len(h.Logs)) + for i, item := range h.Logs { + d.Logs[i] = item.inner + } + return nil +} + +// OpaqueDigestItemID is the type of a digest item that contains raw data; this also names the consensus engine ID where +// applicable. Used to identify one or more digest items of interest. +type OpaqueDigestItemID interface { + isOpaqueDigestItemID() +} + +// OpaqueDigestItemIDPreRuntime is opaque type corresponding to [DigestItemPreRuntime]. +type OpaqueDigestItemIDPreRuntime ConsensusEngineID + +// OpaqueDigestItemIDConsensus is opaque type corresponding to [DigestItemConsensus]. +type OpaqueDigestItemIDConsensus ConsensusEngineID + +// OpaqueDigestItemIDSeal is opaque type corresponding to [DigestItemSeal]. +type OpaqueDigestItemIDSeal ConsensusEngineID + +// OpaqueDigestItemIDOther is some other (non-prescribed) type. +type OpaqueDigestItemIDOther struct{} + +func (OpaqueDigestItemIDPreRuntime) isOpaqueDigestItemID() {} +func (OpaqueDigestItemIDConsensus) isOpaqueDigestItemID() {} +func (OpaqueDigestItemIDSeal) isOpaqueDigestItemID() {} +func (OpaqueDigestItemIDOther) isOpaqueDigestItemID() {} diff --git a/internal/primitives/runtime/digest_test.go b/internal/primitives/runtime/digest_test.go index e8cd90aa55..6fcbbf4b4a 100644 --- a/internal/primitives/runtime/digest_test.go +++ b/internal/primitives/runtime/digest_test.go @@ -13,10 +13,10 @@ import ( func TestEncodeDecodeDigest(t *testing.T) { digest := Digest{ Logs: []DigestItem{ - NewDigestItem(PreRuntime{ + DigestItemPreRuntime{ ConsensusEngineID: ConsensusEngineID{'F', 'R', 'N', 'K'}, Bytes: []byte("test"), - }), + }, }, } diff --git a/internal/primitives/runtime/generic/block.go b/internal/primitives/runtime/generic/block.go index b179aaa984..f18e17103f 100644 --- a/internal/primitives/runtime/generic/block.go +++ b/internal/primitives/runtime/generic/block.go @@ -46,38 +46,38 @@ func (id BlockIDNumber[H]) String() string { } // Block is a block. -type Block[N runtime.Number, H runtime.Hash, Hasher runtime.Hasher[H], E runtime.Extrinsic] struct { +type Block[N runtime.Number, H runtime.Hash, Hasher runtime.Hasher[H], E runtime.Extrinsic, Header runtime.Header[N, H]] struct { // The block header. - header runtime.Header[N, H] + header Header // The accompanying extrinsics. extrinsics []E } // Header returns the header. -func (b Block[N, H, Hasher, E]) Header() runtime.Header[N, H] { +func (b Block[N, H, Hasher, E, Header]) Header() Header { return b.header } // Extrinsics returns the block extrinsics. -func (b Block[N, H, Hasher, E]) Extrinsics() []E { +func (b Block[N, H, Hasher, E, Header]) Extrinsics() []E { return b.extrinsics } // Deconstruct returns both header and extrinsics. -func (b Block[N, H, Hasher, E]) Deconstruct() (header runtime.Header[N, H], extrinsics []E) { +func (b Block[N, H, Hasher, E, Header]) Deconstruct() (header Header, extrinsics []E) { return b.Header(), b.Extrinsics() } // Hash returns the block hash. -func (b Block[N, H, Hasher, E]) Hash() H { +func (b Block[N, H, Hasher, E, Header]) Hash() H { hasher := *new(Hasher) return hasher.HashEncoded(b.header) } // NewBlock is the constructor for Block. -func NewBlock[Hasher runtime.Hasher[H], E runtime.Extrinsic, N runtime.Number, H runtime.Hash]( - header runtime.Header[N, H], extrinsics []E) Block[N, H, Hasher, E] { - return Block[N, H, Hasher, E]{ +func NewBlock[Hasher runtime.Hasher[H], E runtime.Extrinsic, N runtime.Number, H runtime.Hash, Header runtime.Header[N, H]]( + header Header, extrinsics []E) Block[N, H, Hasher, E, Header] { + return Block[N, H, Hasher, E, Header]{ header: header, extrinsics: extrinsics, } @@ -88,8 +88,9 @@ type SignedBlock[ H runtime.Hash, Hasher runtime.Hasher[H], E runtime.Extrinsic, + Header runtime.Header[N, H], ] struct { - Block Block[N, H, Hasher, E] + Block Block[N, H, Hasher, E, Header] Justifications runtime.Justifications } @@ -98,15 +99,16 @@ func NewSignedBlock[ H runtime.Hash, Hasher runtime.Hasher[H], E runtime.Extrinsic, + Header runtime.Header[N, H], ]( - block Block[N, H, Hasher, E], + block Block[N, H, Hasher, E, Header], justifications runtime.Justifications, -) *SignedBlock[N, H, Hasher, E] { - return &SignedBlock[N, H, Hasher, E]{ +) *SignedBlock[N, H, Hasher, E, Header] { + return &SignedBlock[N, H, Hasher, E, Header]{ Block: block, Justifications: justifications, } } -var _ runtime.Block[uint, hash.H256, runtime.OpaqueExtrinsic] = Block[uint, hash.H256, - runtime.BlakeTwo256, runtime.OpaqueExtrinsic]{} +var _ runtime.Block[hash.H256, uint, runtime.OpaqueExtrinsic, Header[uint, hash.H256, runtime.BlakeTwo256]] = Block[uint, hash.H256, + runtime.BlakeTwo256, runtime.OpaqueExtrinsic, Header[uint, hash.H256, runtime.BlakeTwo256]]{} diff --git a/internal/primitives/runtime/generic/header_test.go b/internal/primitives/runtime/generic/header_test.go index d259ac46d7..2e6521dce1 100644 --- a/internal/primitives/runtime/generic/header_test.go +++ b/internal/primitives/runtime/generic/header_test.go @@ -20,10 +20,10 @@ func TestEncodeDecodeHeader(t *testing.T) { hash.H256(""), runtime.Digest{ Logs: []runtime.DigestItem{ - runtime.NewDigestItem(runtime.PreRuntime{ + runtime.DigestItemPreRuntime{ ConsensusEngineID: runtime.ConsensusEngineID{'F', 'R', 'N', 'K'}, Bytes: []byte("test"), - }), + }, }, }, ) diff --git a/internal/primitives/runtime/generic/unchecked_extrinsic.go b/internal/primitives/runtime/generic/unchecked_extrinsic.go new file mode 100644 index 0000000000..3b21153c18 --- /dev/null +++ b/internal/primitives/runtime/generic/unchecked_extrinsic.go @@ -0,0 +1,81 @@ +package generic + +// / Type to represent the version of the [Extension](TransactionExtension) used in this extrinsic. +// pub type ExtensionVersion = u8; +type ExtensionVersion uint8 + +// / Type to represent the extrinsic format version which defines an [UncheckedExtrinsic]. +// pub type ExtrinsicVersion = u8; +type ExtrinsicVersion uint8 + +// / A "header" for extrinsics leading up to the call itself. Determines the type of extrinsic and +// / holds any necessary specialized data. +// #[derive(Eq, PartialEq, Clone)] +// pub enum Preamble { +type Preamble[Address, Signature, Extension any] interface { + isPreamble() +} + +// / An extrinsic without a signature or any extension. This means it's either an inherent or +// / an old-school "Unsigned" (we don't use that terminology any more since it's confusable with +// / the general transaction which is without a signature but does have an extension). +// / +// / NOTE: In the future, once we remove `ValidateUnsigned`, this will only serve Inherent +// / extrinsics and thus can be renamed to `Inherent`. +// Bare(ExtrinsicVersion), +type PreambleBare ExtrinsicVersion + +func (pb PreambleBare) isPreamble() {} + +// / An old-school transaction extrinsic which includes a signature of some hard-coded crypto. +// / Available only on extrinsic version 4. +// Signed(Address, Signature, Extension), +type PreambleSigned[Address, Signature, Extension any] struct { + Address Address + Signature Signature + Extension Extension +} + +func (ps PreambleSigned[Address, Signature, Extension]) isPreamble() {} + +// / A new-school transaction extrinsic which does not include a signature by default. The +// / origin authorization, through signatures or other means, is performed by the transaction +// / extension in this extrinsic. Available starting with extrinsic version 5. +// General(ExtensionVersion, Extension), +type PreambleGeneral[Extension any] struct { + ExtensionVersion ExtensionVersion + Extension Extension +} + +func (pg PreambleGeneral[Extension]) isPreamble() {} + +// pub struct UncheckedExtrinsic { +type UncheckedExtrinsic[Address, Call, Signature, Extension any] struct { + /// Information regarding the type of extrinsic this is (inherent or transaction) as well as + /// associated extension (`Extension`) data if it's a transaction and a possible signature. + // pub preamble: Preamble, + Preamble Preamble[Address, Signature, Extension] + /// The function that should be called. + // pub function: Call, + Function Call +} + +func (ue UncheckedExtrinsic[Address, Call, Signature, Extension]) IsSigned() *bool { + switch ue.Preamble.(type) { + case PreambleSigned[Address, Signature, Extension]: + t := true + return &t + case PreambleGeneral[Extension]: + f := false + return &f + case PreambleBare: + f := false + return &f + default: + return nil + } +} + +func (UncheckedExtrinsic[Address, Call, Signature, Extension]) Bytes() []byte { + panic("unimpl") +} diff --git a/internal/primitives/runtime/interfaces.go b/internal/primitives/runtime/interfaces.go index df270066dd..a4ce666da9 100644 --- a/internal/primitives/runtime/interfaces.go +++ b/internal/primitives/runtime/interfaces.go @@ -5,7 +5,11 @@ package runtime import ( "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/core/hashing" + "github.com/ChainSafe/gossamer/internal/primitives/crypto/hashing" + "github.com/ChainSafe/gossamer/internal/primitives/io/trie" + + "github.com/ChainSafe/gossamer/internal/primitives/kv" + "github.com/ChainSafe/gossamer/internal/primitives/storage" "github.com/ChainSafe/gossamer/pkg/scale" "golang.org/x/exp/constraints" ) @@ -26,6 +30,8 @@ type Hash interface { Length() int } +type KeyValue = kv.KeyValue + // Hasher is an interface around hashing type Hasher[H Hash] interface { // Produce the hash of some byte-slice. @@ -36,6 +42,9 @@ type Hasher[H Hash] interface { // Construct new hash from source data NewHash(data []byte) H + + /// The Patricia tree root of the given mapping. + TrieRoot(input []KeyValue, stateVersion storage.StateVersion) H } // Blake2-256 Hash implementation. @@ -57,6 +66,11 @@ func (bt256 BlakeTwo256) NewHash(data []byte) hash.H256 { return hash.H256(data) } +func (bt256 BlakeTwo256) TrieRoot(input []KeyValue, stateVersion storage.StateVersion) hash.H256 { + return trie.BlakeTwo256Root(input, stateVersion) + panic("unimpl") +} + var _ Hasher[hash.H256] = BlakeTwo256{} // Header is the interface for a header. It has types for a Number, @@ -98,13 +112,13 @@ type Header[N Number, H Hash] interface { // Block represents a block. It has types for Extrinsic pieces of information as well as a Header. // // You can iterate over each of the Extrinsics and retrieve the Header. -type Block[N Number, H Hash, E Extrinsic] interface { +type Block[H Hash, N Number, E Extrinsic, Head Header[N, H]] interface { // Returns a reference to the header. - Header() Header[N, H] + Header() Head // Returns a reference to the list of extrinsics. Extrinsics() []E // Split the block into header and list of extrinsics. - Deconstruct() (header Header[N, H], extrinsics []E) + Deconstruct() (header Head, extrinsics []E) // Returns the hash of the block. Hash() H } diff --git a/internal/primitives/runtime/runtime.go b/internal/primitives/runtime/runtime.go index 8fd08da542..fa26a5528a 100644 --- a/internal/primitives/runtime/runtime.go +++ b/internal/primitives/runtime/runtime.go @@ -3,7 +3,11 @@ package runtime -import "slices" +import ( + "slices" + + "github.com/ChainSafe/gossamer/internal/primitives/storage" +) // Justification is an abstraction over justification for a block's validity under a consensus algorithm. // @@ -43,6 +47,37 @@ func (j Justifications) Get(engineID ConsensusEngineID) *EncodedJustification { return nil } +// / Return a copy of the encoded justification for the given consensus +// / engine, if it exists. +// +// pub fn into_justification(self, engine_id: ConsensusEngineId) -> Option { +// self.into_iter().find(|j| j.0 == engine_id).map(|j| j.1) +// } +func (j Justifications) IntoJustification(enginedID ConsensusEngineID) *EncodedJustification { + for _, justification := range j { + if justification.ConsensusEngineID == enginedID { + return &justification.EncodedJustification + } + } + return nil +} + +// // / Complex storage builder stuff. +// // #[cfg(feature = "std")] +// // pub trait BuildStorage { +type BuildStorage interface { + // /// Build the storage out of this builder. + // fn build_storage(&self) -> Result { + // let mut storage = Default::default(); + // self.assimilate_storage(&mut storage)?; + // Ok(storage) + // } + BuildStorage() (storage.Storage, error) + // + // /// Assimilate the storage for this module into pre-existing overlays. + // fn assimilate_storage(&self, storage: &mut sp_core::storage::Storage) -> Result<(), String>; +} + // EncodedJustification returns a copy of the encoded justification for the given consensus engine, if it exists func (j Justifications) EncodedJustification(engineID ConsensusEngineID) *EncodedJustification { for _, justification := range j { diff --git a/internal/primitives/state-machine/basic/basic.go b/internal/primitives/state-machine/basic/basic.go index 1967135cfb..55814a9480 100644 --- a/internal/primitives/state-machine/basic/basic.go +++ b/internal/primitives/state-machine/basic/basic.go @@ -10,6 +10,7 @@ import ( "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" "github.com/ChainSafe/gossamer/internal/primitives/externalities" + "github.com/ChainSafe/gossamer/internal/primitives/kv" "github.com/ChainSafe/gossamer/internal/primitives/runtime" statemachine "github.com/ChainSafe/gossamer/internal/primitives/state-machine" "github.com/ChainSafe/gossamer/internal/primitives/state-machine/overlayedchanges" @@ -17,8 +18,6 @@ import ( "github.com/ChainSafe/gossamer/internal/primitives/storage/keys" "github.com/ChainSafe/gossamer/internal/primitives/trie" "github.com/ChainSafe/gossamer/pkg/scale" - "github.com/ChainSafe/gossamer/pkg/trie/db" - "github.com/ChainSafe/gossamer/pkg/trie/inmemory" "github.com/tidwall/btree" ) @@ -211,15 +210,13 @@ func (be *BasicExternalities) StorageAppend(key []byte, value []byte) { } func (be *BasicExternalities) StorageRoot(stateVersion storage.StateVersion) []byte { - memDB := db.NewEmptyMemoryDB() - storageTrie := inmemory.NewTrie(nil, memDB) + // memDB := db.NewEmptyMemoryDB() + // storageTrie := inmemory.NewTrie(nil, memDB) + var top btree.Map[string, []byte] for k, v := range be.overlay.Changes() { if v.Value() != nil { - err := storageTrie.Put([]byte(k), []byte(v.Value())) - if err != nil { - panic("error building trie to calculate storage root") - } + top.Set(string(k), v.Value()) } } @@ -230,19 +227,34 @@ func (be *BasicExternalities) StorageRoot(stateVersion storage.StateVersion) []b for _, childInfo := range be.overlay.Children() { childRoot := be.ChildStorageRoot(childInfo, stateVersion) - var err error + // var err error if bytes.Equal(emptyHash.Bytes(), childRoot) { - err = storageTrie.Delete(childInfo.PrefixedStorageKey()) + // err = storageTrie.Delete(childInfo.PrefixedStorageKey()) + top.Delete(string(childInfo.PrefixedStorageKey())) } else { - err = storageTrie.Put(childInfo.PrefixedStorageKey(), childRoot) + // err = storageTrie.Put(childInfo.PrefixedStorageKey(), childRoot) + top.Set(string(childInfo.PrefixedStorageKey()), childRoot) } + // if err != nil { + // panic("unexpected error updating child trie key") + // } + } - if err != nil { - panic("unexpected error updating child trie key") - } + topKVs := make([]kv.KeyValue, 0, top.Len()) + keys, values := top.KeyValues() + for i, key := range keys { + value := values[i] + topKVs = append(topKVs, kv.KeyValue{Key: []byte(key), Value: value}) } - return stateVersion.TrieLayout().MustHash(storageTrie).ToBytes() + switch stateVersion { + case storage.StateVersionV0: + return trie.LayoutV0[runtime.BlakeTwo256, hash.H256]{}.TrieRoot(topKVs).Bytes() + case storage.StateVersionV1: + return trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{}.TrieRoot(topKVs).Bytes() + default: + panic("unreachable") + } } func (be *BasicExternalities) ChildStorageRoot( diff --git a/internal/primitives/state-machine/trie_backend_essence.go b/internal/primitives/state-machine/trie_backend_essence.go index 3da8e5be58..f7b452daf5 100644 --- a/internal/primitives/state-machine/trie_backend_essence.go +++ b/internal/primitives/state-machine/trie_backend_essence.go @@ -16,7 +16,6 @@ import ( "github.com/ChainSafe/gossamer/internal/primitives/trie" "github.com/ChainSafe/gossamer/internal/primitives/trie/cache" "github.com/ChainSafe/gossamer/internal/primitives/trie/recorder" - ptrie "github.com/ChainSafe/gossamer/pkg/trie" triedb "github.com/ChainSafe/gossamer/pkg/trie/triedb" "golang.org/x/exp/constraints" ) @@ -266,8 +265,7 @@ func withTrieDB[H runtime.Hash, Hasher runtime.Hasher[H]]( withRecorderAndCache(tbe, &root, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) { trieDB := triedb.NewTrieDB( - root, db, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(ptrie.V1) + root, db, trie.LayoutV1[Hasher, H]{}, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) callback(trieDB) }) } @@ -357,7 +355,7 @@ func (tbe *trieBackendEssence[H, Hasher]) NextStorageKeyFromRoot( // Get the value of storage at given key. func (tbe *trieBackendEssence[H, Hasher]) Storage(key []byte) (val StorageValue, err error) { withRecorderAndCache(tbe, nil, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) { - val, err = trie.ReadTrieValue[H, Hasher](tbe, tbe.root, key, recorder, cache, triedb.V1) + val, err = trie.ReadTrieValue[H, Hasher](tbe, tbe.root, key, recorder, cache, trie.LayoutV1[Hasher, H]{}) }) return } @@ -366,8 +364,7 @@ func (tbe *trieBackendEssence[H, Hasher]) Storage(key []byte) (val StorageValue, func (tbe *trieBackendEssence[H, Hasher]) StorageHash(key []byte) (hash *H, err error) { withRecorderAndCache[H, Hasher](tbe, nil, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) { trieDB := triedb.NewTrieDB( - tbe.root, tbe, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(ptrie.V1) + tbe.root, tbe, trie.LayoutV1[Hasher, H]{}, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) hash, err = trieDB.GetHash(key) }) return @@ -395,7 +392,7 @@ func (tbe *trieBackendEssence[H, Hasher]) ChildStorage(childInfo storage.ChildIn key, recorder, cache, - triedb.V1, + trie.LayoutV1[Hasher, H]{}, ) }) return val, err @@ -422,7 +419,7 @@ func (tbe *trieBackendEssence[H, Hasher]) ChildStorageHash(childInfo storage.Chi key, recorder, cache, - triedb.V1, + trie.LayoutV1[Hasher, H]{}, ) }) return hash, err @@ -431,7 +428,7 @@ func (tbe *trieBackendEssence[H, Hasher]) ChildStorageHash(childInfo storage.Chi // Get the closest merkle value at given key. func (tbe *trieBackendEssence[H, Hasher]) ClosestMerkleValue(key []byte) (val triedb.MerkleValue[H], err error) { withRecorderAndCache(tbe, nil, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) { - val, err = trie.ReadTrieFirstDescendantValue[H, Hasher](tbe, tbe.root, key, recorder, cache, triedb.V1) + val, err = trie.ReadTrieFirstDescendantValue[H, Hasher](tbe, tbe.root, key, recorder, cache, trie.LayoutV1[Hasher, H]{}) }) return } @@ -452,7 +449,7 @@ func (tbe *trieBackendEssence[H, Hasher]) ChildClosestMerkleValue( withRecorderAndCache(tbe, &childRoot, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) { val, err = trie.ReadChildTrieFirstDescendantValue[H, Hasher]( - childInfo.Keyspace(), tbe, tbe.root, key, recorder, cache, triedb.V1) + childInfo.Keyspace(), tbe, tbe.root, key, recorder, cache, trie.LayoutV1[Hasher, H]{}) }) return } @@ -525,7 +522,14 @@ func (tbe *trieBackendEssence[H, Hasher]) StorageRoot( root := withRecorderAndCacheForStorageRoot( tbe, nil, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) (*H, H) { eph := newEphemeral[H, Hasher](tbe.BackendStorage(), writeOverlay) - root, err := trie.DeltaTrieRoot[H, Hasher](eph, tbe.root, delta, recorder, cache, stateVersion.TrieLayout()) + var layout triedb.TrieLayout + switch stateVersion { + case storage.StateVersionV1: + layout = trie.LayoutV1[Hasher, H]{} + case storage.StateVersionV0: + layout = trie.LayoutV0[Hasher, H]{} + } + root, err := trie.DeltaTrieRoot[H, Hasher](eph, tbe.root, delta, recorder, cache, layout) if err != nil { log.Printf("WARN: failed to write to trie: %v", err) return nil, tbe.root @@ -558,8 +562,15 @@ func (tbe *trieBackendEssence[H, Hasher]) ChildStorageRoot( newChildRoot := withRecorderAndCacheForStorageRoot( tbe, &childRoot, func(recorder triedb.TrieRecorder, cache triedb.TrieCache[H]) (*H, H) { eph := newEphemeral(tbe.BackendStorage(), writeOverlay) + var layout triedb.TrieLayout + switch stateVersion { + case storage.StateVersionV1: + layout = trie.LayoutV1[Hasher, H]{} + case storage.StateVersionV0: + layout = trie.LayoutV0[Hasher, H]{} + } root, err := trie.ChildDeltaTrieRoot[H, Hasher]( - childInfo.Keyspace(), eph, childRoot, delta, recorder, cache, stateVersion.TrieLayout()) + childInfo.Keyspace(), eph, childRoot, delta, recorder, cache, layout) if err != nil { log.Printf("WARN: Failed to write to trie: %v", err) return nil, childRoot diff --git a/internal/primitives/state-machine/trie_backend_essence_test.go b/internal/primitives/state-machine/trie_backend_essence_test.go index 686e21aa09..47cc273592 100644 --- a/internal/primitives/state-machine/trie_backend_essence_test.go +++ b/internal/primitives/state-machine/trie_backend_essence_test.go @@ -21,6 +21,8 @@ var ( _ hashdb.HashDB[hash.H256] = &ephemeral[hash.H256, runtime.BlakeTwo256]{} ) +var layoutV1 = trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{} + func TestTrieBackendEssence(t *testing.T) { t.Run("next_storage_key_and_next_child_storage_key_work", func(t *testing.T) { childInfo := storage.NewDefaultChildInfo([]byte("MyChild")) @@ -31,8 +33,7 @@ func TestTrieBackendEssence(t *testing.T) { mdb := trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256]() { - trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](mdb) - trie.SetVersion(triedb.V1) + trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](mdb, layoutV1) require.NoError(t, trie.Set([]byte("3"), []byte{1})) require.NoError(t, trie.Set([]byte("4"), []byte{1})) require.NoError(t, trie.Set([]byte("6"), []byte{1})) @@ -43,8 +44,7 @@ func TestTrieBackendEssence(t *testing.T) { ksdb := trie.NewKeyspacedDB(mdb, childInfo.Keyspace()) // reuse of root_1 implicitly assert child trie root is same // as top trie (contents must remain the same). - trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](ksdb) - trie.SetVersion(triedb.V1) + trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](ksdb, layoutV1) err := trie.Set([]byte("3"), []byte{1}) require.NoError(t, err) require.NoError(t, trie.Set([]byte("3"), []byte{1})) @@ -54,8 +54,7 @@ func TestTrieBackendEssence(t *testing.T) { require.Equal(t, root1, root) } { - trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](mdb) - trie.SetVersion(triedb.V1) + trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](mdb, layoutV1) bleh := childInfo.PrefixedStorageKey() require.NoError(t, trie.Set(slices.Clone(bleh), root1.Bytes())) root2 = trie.MustHash() diff --git a/internal/primitives/state-machine/trie_backend_test.go b/internal/primitives/state-machine/trie_backend_test.go index 7f8bc88b7d..7868b56d79 100644 --- a/internal/primitives/state-machine/trie_backend_test.go +++ b/internal/primitives/state-machine/trie_backend_test.go @@ -34,11 +34,17 @@ func testDB( childInfo := storage.NewDefaultChildInfo(ChildKey1) mdb := trie.NewPrefixedMemoryDB[hash.H256, runtime.BlakeTwo256]() var root hash.H256 + var layout trie.Layout[hash.H256] + switch stateVersion { + case storage.StateVersionV1: + layout = trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{} + case storage.StateVersionV0: + layout = trie.LayoutV0[runtime.BlakeTwo256, hash.H256]{} + } { ksdb := trie.NewKeyspacedDB(mdb, childInfo.Keyspace()) - trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](ksdb) - trie.SetVersion(stateVersion.TrieLayout()) + trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](ksdb, layout) require.NoError(t, trie.Set([]byte("value3"), bytes.Repeat([]byte{142}, 33))) require.NoError(t, trie.Set([]byte("value4"), bytes.Repeat([]byte{124}, 33))) root = trie.MustHash() @@ -58,8 +64,7 @@ func testDB( } } - trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](mdb) - trie.SetVersion(stateVersion.TrieLayout()) + trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](mdb, layout) build(trie, childInfo, subRoot) root = trie.MustHash() } @@ -322,7 +327,14 @@ func TestTrieBackend(t *testing.T) { cache = &local } testDB, testRoot := testDB(t, param.StateVersion) - iter, err := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](testRoot, testDB).Iterator() + var layout trie.Layout[hash.H256] + switch param.StateVersion { + case storage.StateVersionV1: + layout = trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{} + case storage.StateVersionV0: + layout = trie.LayoutV0[runtime.BlakeTwo256, hash.H256]{} + } + iter, err := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](testRoot, testDB, layout).Iterator() require.NoError(t, err) expected := make([][]byte, 0) for { diff --git a/internal/primitives/storage/storage.go b/internal/primitives/storage/storage.go index 38a3a0a05d..b118f9432d 100644 --- a/internal/primitives/storage/storage.go +++ b/internal/primitives/storage/storage.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/ChainSafe/gossamer/internal/primitives/storage/keys" - "github.com/ChainSafe/gossamer/pkg/trie" "github.com/tidwall/btree" ) @@ -170,16 +169,4 @@ const ( StateVersionV0 StateVersion = iota StateVersionV1 ) - const DefaultStateVersion StateVersion = StateVersionV1 - -func (svv StateVersion) TrieLayout() trie.TrieLayout { - switch svv { - case StateVersionV0: - return trie.V0 - case StateVersionV1: - return trie.V1 - default: - panic("unreachable") - } -} diff --git a/internal/primitives/trie/cache/cache_test.go b/internal/primitives/trie/cache/cache_test.go index 59c4ab1151..9f66f63f8c 100644 --- a/internal/primitives/trie/cache/cache_test.go +++ b/internal/primitives/trie/cache/cache_test.go @@ -12,7 +12,6 @@ import ( "github.com/ChainSafe/gossamer/internal/primitives/runtime" "github.com/ChainSafe/gossamer/internal/primitives/trie" "github.com/ChainSafe/gossamer/internal/primitives/trie/recorder" - pkgtrie "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/pkg/trie/triedb" "github.com/stretchr/testify/require" ) @@ -26,12 +25,16 @@ var testData = []triedb.TrieItem{ const cacheSize uint = 1024 * 10 +var ( + layoutV0 = trie.LayoutV0[runtime.BlakeTwo256, hash.H256]{} + layoutV1 = trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{} +) + func createTrie(t *testing.T) (*trie.MemoryDB[hash.H256, runtime.BlakeTwo256], hash.H256) { t.Helper() db := trie.NewMemoryDB[hash.H256, runtime.BlakeTwo256]() - trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(pkgtrie.V1) + trie := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db, layoutV1) for _, item := range testData { err := trie.Set(item.Key, item.Value) require.NoError(t, err) @@ -50,8 +53,7 @@ func Test_SharedTrieCache(t *testing.T) { { cache, unlock := localCache.TrieCache(root) trie := triedb.NewTrieDB( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) val, err := triedb.GetWith(trie, testData[0].Key, func(d []byte) []byte { return d }) require.NoError(t, err) @@ -98,8 +100,7 @@ func Test_SharedTrieCache(t *testing.T) { { cache, unlock := localCache.TrieCache(root) trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) // We should now get the "fake_data", because we inserted this manually to the cache. val, err := triedb.GetWith(trie, testData[1].Key, func(d []byte) []byte { return d }) @@ -126,8 +127,7 @@ func Test_SharedTrieCache(t *testing.T) { { trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) require.NoError(t, trie.Set(newKey, newValue)) newRoot = trie.MustHash() } @@ -168,7 +168,7 @@ func Test_SharedTrieCache(t *testing.T) { cache, unlock := localCache.TrieCache(root) recorder := recorder.TrieRecorder(root) trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, + root, db, layoutV0, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache), triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), ) @@ -189,7 +189,7 @@ func Test_SharedTrieCache(t *testing.T) { memoryDB := trie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) { - trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memoryDB) + trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memoryDB, layoutV0) for _, item := range testData { val, err := triedb.GetWith(trie, item.Key, func(d []byte) []byte { return d }) require.NoError(t, err) @@ -230,7 +230,7 @@ func Test_SharedTrieCache(t *testing.T) { recorder := recorder.TrieRecorder(root) trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, &db, + root, &db, layoutV0, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache), triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), ) @@ -248,7 +248,7 @@ func Test_SharedTrieCache(t *testing.T) { memoryDB := trie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) var proofRoot hash.H256 { - trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memoryDB) + trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memoryDB, layoutV0) for _, item := range dataToAdd { err := trie.Set(item.Key, item.Value) require.NoError(t, err) @@ -268,8 +268,7 @@ func Test_SharedTrieCache(t *testing.T) { cache, unlock := localCache.TrieCache(root) trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) for _, item := range testData { val, err := triedb.GetWith(trie, item.Key, func(d []byte) []byte { return d }) @@ -308,8 +307,7 @@ func Test_SharedTrieCache(t *testing.T) { localCache := sharedCache.LocalTrieCache() cache, unlock := localCache.TrieCache(root) trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) for _, item := range testData[:2] { val, err := triedb.GetWith(trie, item.Key, func(d []byte) []byte { return d }) @@ -342,8 +340,7 @@ func Test_SharedTrieCache(t *testing.T) { localCache := sharedCache.LocalTrieCache() cache, unlock := localCache.TrieCache(root) trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) for _, item := range testData[2:] { val, err := triedb.GetWith(trie, item.Key, func(d []byte) []byte { return d }) @@ -374,8 +371,7 @@ func Test_SharedTrieCache(t *testing.T) { cache, unlock := localCache.TrieCache(root) { trie := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, db, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(pkgtrie.V1) + root, db, layoutV1, triedb.WithCache[hash.H256, runtime.BlakeTwo256](cache)) value := bytes.Repeat([]byte{10}, 100) // Ensure we add enough data that would overflow the cache. for i := 0; i < (int(cacheSize) / 100 * 2); i++ { diff --git a/internal/primitives/trie/node_header.go b/internal/primitives/trie/node_header.go new file mode 100644 index 0000000000..6e59ce5dae --- /dev/null +++ b/internal/primitives/trie/node_header.go @@ -0,0 +1,82 @@ +package trie + +import "github.com/ChainSafe/gossamer/internal/saturating" + +// / NodeHeader without content +type nodeKind uint + +const ( + nodeKindLeaf nodeKind = iota + nodeKindBranchNoValue + nodeKindBranchWithValue + nodeKindHashedValueLeaf + nodeKindHashedValueBranch +) + +// / Returns an iterator over encoded bytes for node header and size. +// / Size encoding allows unlimited, length inefficient, representation, but +// / is bounded to 16 bit maximum value to avoid possible DOS. +// pub(crate) fn size_and_prefix_iterator( +// +// size: usize, +// prefix: u8, +// prefix_mask: usize, +// +// ) -> impl Iterator { +func sizeAndPrefixIterator(size uint, prefix uint8, prefixMask int) []byte { + // let max_value = 255u8 >> prefix_mask; + maxValue := uint8(255) >> prefixMask + // let l1 = core::cmp::min((max_value as usize).saturating_sub(1), size); + l1 := saturating.Sub(maxValue, 1) + if size < uint(l1) { + l1 = uint8(size) + } + // + // let (first_byte, mut rem) = if size == l1 { + // (once(prefix + l1 as u8), 0) + // } else { + // + // (once(prefix + max_value as u8), size - l1) + // }; + var firstByte uint8 + var rem uint + if size == uint(l1) { + firstByte = prefix + l1 + rem = 0 + } else { + firstByte = prefix + maxValue + rem = size - uint(l1) + } + // + // let next_bytes = move || { + // if rem > 0 { + // if rem < 256 { + // let result = rem - 1; + // rem = 0; + // Some(result as u8) + // } else { + // rem = rem.saturating_sub(255); + // Some(255) + // } + // } else { + // None + // } + // }; + nextBytes := make([]byte, 0) + for { + if rem > 0 { + if rem < 256 { + result := rem - 1 + rem = 0 + nextBytes = append(nextBytes, uint8(result)) + } else { + rem = saturating.Sub(rem, 255) + nextBytes = append(nextBytes, 255) + } + } else { + break + } + } + + return append([]byte{firstByte}, nextBytes...) +} diff --git a/internal/primitives/trie/recorder/recorder_test.go b/internal/primitives/trie/recorder/recorder_test.go index 7c66a7b92a..11c11ef07a 100644 --- a/internal/primitives/trie/recorder/recorder_test.go +++ b/internal/primitives/trie/recorder/recorder_test.go @@ -9,8 +9,7 @@ import ( memorydb "github.com/ChainSafe/gossamer/internal/memory-db" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" "github.com/ChainSafe/gossamer/internal/primitives/runtime" - ptrie "github.com/ChainSafe/gossamer/internal/primitives/trie" - "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/internal/primitives/trie" "github.com/ChainSafe/gossamer/pkg/trie/triedb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,7 +24,7 @@ func makeValue(i uint8) []byte { return val } -var testData []ptrie.KeyValue = []ptrie.KeyValue{ +var testData []trie.KeyValue = []trie.KeyValue{ { Key: []byte("key1"), Value: makeValue(1), @@ -44,6 +43,11 @@ var testData []ptrie.KeyValue = []ptrie.KeyValue{ }, } +var ( + layoutV0 = trie.LayoutV0[runtime.BlakeTwo256, hash.H256]{} + layoutV1 = trie.LayoutV1[runtime.BlakeTwo256, hash.H256]{} +) + type MemoryDB = memorydb.MemoryDB[ hash.H256, runtime.BlakeTwo256, hash.H256, memorydb.HashKey[hash.H256], ] @@ -58,8 +62,7 @@ func newMemoryDB() *MemoryDB { func createTrie(t *testing.T) (db *MemoryDB, root hash.H256) { t.Helper() db = newMemoryDB() - trieDB := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db, layoutV1) for _, td := range testData { err := trieDB.Set(td.Key, td.Value) @@ -77,19 +80,17 @@ func TestRecorder(t *testing.T) { { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := trieDB.Get(testData[0].Key) require.NoError(t, err) require.Equal(t, testData[0].Value, val) } storageProof := rec.DrainStorageProof() - memDB := ptrie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) + memDB := trie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) // Check that we recorded the required data - trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB, layoutV1) val, err := trieDB.Get(testData[0].Key) require.NoError(t, err) require.Equal(t, testData[0].Value, val) @@ -127,8 +128,7 @@ func TestRecorder_TransactionsRollback(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := trieDB.Get(testData[i].Key) require.NoError(t, err) require.Equal(t, testData[i].Value, val) @@ -142,10 +142,9 @@ func TestRecorder_TransactionsRollback(t *testing.T) { assert.Equal(t, stats[4-i], newRecorderStats(&rec)) storageProof := rec.StorageProof() - memDB := ptrie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) + memDB := trie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) - trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB, layoutV1) // Check that the required data is still present. for a := 0; a < 4; a++ { @@ -177,8 +176,7 @@ func TestRecorder_TransactionsCommit(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := trieDB.Get(testData[i].Key) assert.NoError(t, err) assert.Equal(t, testData[i].Value, val) @@ -196,11 +194,10 @@ func TestRecorder_TransactionsCommit(t *testing.T) { assert.Equal(t, stats, newRecorderStats(&rec)) storageProof := rec.StorageProof() - memDB := ptrie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) + memDB := trie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) // Check that we recorded the required data - trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB, layoutV1) // Check that the required data is still present. for i := 0; i < 4; i++ { @@ -218,8 +215,7 @@ func TestRecorder_TransactionsCommitAndRollback(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := trieDB.Get(testData[i].Key) assert.NoError(t, err) assert.Equal(t, testData[i].Value, val) @@ -233,8 +229,7 @@ func TestRecorder_TransactionsCommitAndRollback(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := trieDB.Get(testData[i].Key) assert.NoError(t, err) assert.Equal(t, testData[i].Value, val) @@ -254,11 +249,10 @@ func TestRecorder_TransactionsCommitAndRollback(t *testing.T) { assert.Equal(t, 0, len(rec.inner.transactions)) storageProof := rec.StorageProof() - memDB := ptrie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) + memDB := trie.NewMemoryDBFromStorageProof[hash.H256, runtime.BlakeTwo256](storageProof) // Check that we recorded the required data - trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB[hash.H256, runtime.BlakeTwo256](root, memDB, layoutV1) // Check that the required data is still present. for i := 0; i < 4; i++ { @@ -287,8 +281,7 @@ func TestRecorder_TransactionAccessedKeys(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) hash, err := trieDB.GetHash(key) require.NoError(t, err) @@ -301,8 +294,7 @@ func TestRecorder_TransactionAccessedKeys(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := triedb.GetWith(trieDB, testData[0].Key, func(data []byte) []byte { return data }) require.NoError(t, err) @@ -329,8 +321,7 @@ func TestRecorder_TransactionAccessedKeys(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) val, err := triedb.GetWith(trieDB, testData[0].Key, func(data []byte) []byte { return data }) require.NoError(t, err) @@ -343,8 +334,7 @@ func TestRecorder_TransactionAccessedKeys(t *testing.T) { rec.StartTransaction() { trieRecorder := rec.TrieRecorder(root) - trieDB := triedb.NewTrieDB(root, db, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) - trieDB.SetVersion(trie.V1) + trieDB := triedb.NewTrieDB(root, db, layoutV1, triedb.WithRecorder[hash.H256, runtime.BlakeTwo256](trieRecorder)) hash, err := trieDB.GetHash(testData[0].Key) require.NoError(t, err) diff --git a/internal/primitives/trie/storage_proof.go b/internal/primitives/trie/storage_proof.go index 104ef51a65..7a8d534529 100644 --- a/internal/primitives/trie/storage_proof.go +++ b/internal/primitives/trie/storage_proof.go @@ -5,7 +5,6 @@ package trie import ( hashdb "github.com/ChainSafe/gossamer/internal/hash-db" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" "github.com/tidwall/btree" ) @@ -47,7 +46,7 @@ func (sp *StorageProof) Nodes() [][]byte { } // NewMemoryDBFromStorageProof constructs a [MemoryDB] from a [StorageProof] -func NewMemoryDBFromStorageProof[H runtime.Hash, Hasher runtime.Hasher[H]](sp StorageProof) *MemoryDB[H, Hasher] { +func NewMemoryDBFromStorageProof[H hashdb.Hash, Hasher hashdb.Hasher[H]](sp StorageProof) *MemoryDB[H, Hasher] { db := NewMemoryDB[H, Hasher]() sp.trieNodes.Scan(func(v string) bool { db.Insert(hashdb.EmptyPrefix, []byte(v)) diff --git a/internal/primitives/trie/trie-root/trie_root.go b/internal/primitives/trie/trie-root/trie_root.go new file mode 100644 index 0000000000..a1801b888c --- /dev/null +++ b/internal/primitives/trie/trie-root/trie_root.go @@ -0,0 +1,411 @@ +package trieroot + +import ( + hashdb "github.com/ChainSafe/gossamer/internal/hash-db" + "github.com/ChainSafe/gossamer/internal/primitives/kv" + "github.com/tidwall/btree" +) + +type KeyValue = kv.KeyValue + +// / Different possible value to use for node encoding. +// #[derive(Clone)] +// +// pub enum Value<'a> { +// /// Contains a full value. +// Inline(&'a [u8]), +// /// Contains hash of a value. +// Node(Vec), +// } +type Value interface { + isValue() +} +type ( + /// Contains a full value. + InlineValue []byte + /// Contains hash of a value. + NodeValue []byte +) + +func (InlineValue) isValue() {} +func (NodeValue) isValue() {} + +// fn new(value: &'a [u8], threshold: Option) -> Value<'a> { +func NewValue[Hasher hashdb.Hasher[H], H hashdb.Hash](value []byte, threshold *uint32) Value { + // if let Some(threshold) = threshold { + // if value.len() >= threshold as usize { + // Value::Node(H::hash(value).as_ref().to_vec()) + // } else { + // Value::Inline(value) + // } + // } else { + // + // Value::Inline(value) + // } + if threshold != nil && len(value) >= int(*threshold) { + h := (*new(Hasher)).Hash(value) + return NodeValue(h.Bytes()) + } + return InlineValue(value) +} + +// / Byte-stream oriented trait for constructing closed-form tries. +// pub trait TrieStream { +type TrieStream interface { + /// Construct a new `TrieStream` + // fn new() -> Self; + New() TrieStream + /// Append an Empty node + AppendEmptyData() + /// Start a new Branch node, possibly with a value; takes a list indicating + /// which slots in the Branch node has further child nodes. + BeginBranch( + maybeKey []byte, + maybeValue Value, + hasChildren []bool, + ) + /// Append an empty child node. Optional. + AppendEmptyChild() + /// Wrap up a Branch node portion of a `TrieStream` and append the value + /// stored on the Branch (if any). + EndBranch(value Value) + /// Append a Leaf node + // fn append_leaf(&mut self, key: &[u8], value: Value); + AppendLeaf(key []byte, value Value) + // /// Append an Extension node + // fn append_extension(&mut self, key: &[u8]); + AppendExtension(key []byte) + // /// Append a Branch of Extension substream + // fn append_substream(&mut self, other: Self); + AppendSubstream(other TrieStream) + /// Return the finished `TrieStream` as a vector of bytes. + Out() []byte +} + +// fn shared_prefix_length(first: &[T], second: &[T]) -> usize { +func sharedPrefixLength(first []byte, second []byte) uint { + // first + // + // .iter() + // .zip(second.iter()) + // .position(|(f, s)| f != s) + // .unwrap_or_else(|| cmp::min(first.len(), second.len())) + length := len(first) + if len(second) < length { + length = len(second) + } + var position int = -1 + for i := 0; i < length; i++ { + var ( + a byte + b byte + ) + if i < len(first) { + a = first[i] + } + if i < len(second) { + b = second[i] + } + if a != b { + position = i + break + } + } + if position == -1 { + position = len(first) + if len(second) < position { + position = len(second) + } + } + return uint(position) +} + +// fn trie_root_inner(input: I, no_extension: bool, threshold: Option) -> H::Out +// where +// +// I: IntoIterator, +// A: AsRef<[u8]> + Ord, +// B: AsRef<[u8]>, +// H: Hasher, +// S: TrieStream, +// +// { +func TrieRoot[Hasher hashdb.Hasher[H], H hashdb.Hash]( + input []KeyValue, + threshold *uint32, + stream TrieStream, +) H { + + // // first put elements into btree to sort them and to remove duplicates + // let input = input.into_iter().collect::>(); + inputMap := btree.Map[string, []byte]{} + for _, kv := range input { + inputMap.Set(string(kv.Key), kv.Value) + } + + // // convert to nibbles + // let mut nibbles = Vec::with_capacity(input.keys().map(|k| k.as_ref().len()).sum::() * 2); + // let mut lens = Vec::with_capacity(input.len() + 1); + // lens.push(0); + // for k in input.keys() { + // for &b in k.as_ref() { + // nibbles.push(b >> 4); + // nibbles.push(b & 0x0F); + // } + // lens.push(nibbles.len()); + // } + nibblesLen := 0 + for _, kv := range input { + nibblesLen += len(kv.Key) * 2 + } + nibbles := make([]byte, 0, nibblesLen) + lens := make([]uint, 0, len(input)+1) + lens = append(lens, 0) + for _, kv := range input { + for _, b := range kv.Key { + nibbles = append(nibbles, b>>4) + nibbles = append(nibbles, b&0x0F) + } + lens = append(lens, uint(len(nibbles))) + } + + // // then move them to a vector + // let input = input + // .into_iter() + // .zip(lens.windows(2)) + // .map(|((_, v), w)| (&nibbles[w[0]..w[1]], v)) + // .collect::>(); + trieInput := make([]KeyValue, len(input)) + for i, kv := range input { + in := KeyValue{ + Key: nibbles[lens[i]:lens[i+1]], + Value: kv.Value, + } + trieInput[i] = in + } + + // let mut stream = S::new(); + + // build_trie::(&input, 0, &mut stream, no_extension, threshold); + buildTrie[Hasher, H](trieInput, 0, stream, threshold) + // H::hash(&stream.out()) + return (*new(Hasher)).Hash(stream.Out()) +} + +func buildTrie[Hasher hashdb.Hasher[H], H hashdb.Hash]( + input []KeyValue, + cursor uint, + stream TrieStream, + threshold *uint32, +) { + // match input.len() { + switch len(input) { + case 0: + // No input, just append empty data. + // 0 => stream.append_empty_data(), + stream.AppendEmptyData() + + case 1: + // Leaf node; append the remainder of the key and the value. Done. + // let value = Value::new::(input[0].1.as_ref(), threshold); + // stream.append_leaf(&input[0].0.as_ref()[cursor..], value) + value := NewValue[Hasher](input[0].Value, threshold) + stream.AppendLeaf(input[0].Key[cursor:], value) + + // _ => { + default: + // We have multiple items in the input. Figure out if we should add an + // extension node or a branch node. + // let (key, value) = (&input[0].0.as_ref(), input[0].1.as_ref()); + key := input[0].Key + value := input[0].Value + // Count the number of nibbles in the other elements that are + // shared with the first key. + // e.g. input = [ [1'7'3'10'12'13], [1'7'3'], [1'7'7'8'9'] ] => [1'7'] is common => 2 + // let shared_nibble_count = input.iter().skip(1).fold(key.len(), |acc, &(ref k, _)| { + // cmp::min(shared_prefix_length(key, k.as_ref()), acc) + // }); + sharedNibbleCount := uint(len(key)) + for _, kv := range input[1:] { + sharedPrefixLength := sharedPrefixLength(key, kv.Key) + if sharedPrefixLength < sharedNibbleCount { + sharedNibbleCount = sharedPrefixLength + } + } + // Add an extension node if the number of shared nibbles is greater + // than what we saw on the last call (`cursor`): append the new part + // of the path then recursively append the remainder of all items + // who had this partial key. + // let (cursor, o_branch_slice) = if no_extension { + // if shared_nibble_count > cursor { + // (shared_nibble_count, Some(&key[cursor..shared_nibble_count])) + // } else { + // (cursor, Some(&key[0..0])) + // } + // } else if shared_nibble_count > cursor { + // stream.append_extension(&key[cursor..shared_nibble_count]); + // build_trie_trampoline::( + // input, + // shared_nibble_count, + // stream, + // no_extension, + // threshold, + // ); + // return + // } else { + // (cursor, None) + // }; + + var oBranchSlice []byte + + if sharedNibbleCount > cursor { + oBranchSlice = key[cursor:sharedNibbleCount] + cursor = sharedNibbleCount + } else { + cursor = cursor + oBranchSlice = key[0:0] + } + + // We'll be adding a branch node because the path is as long as it gets. + // First we need to figure out what entries this branch node will have... + + // We have a a value for exactly this key. Branch node will have a value + // attached to it. + // let value = if cursor == key.len() { Some(value) } else { None }; + var val []byte + if cursor == uint(len(key)) { + val = value + } else { + val = nil + } + + // We need to know how many key nibbles each of the children account for. + // let mut shared_nibble_counts = [0usize; 16]; + sharedNibblesCounts := make([]uint, 16) + { + // If the Branch node has a value then the first of the input keys + // is exactly the key for that value and we don't care about it + // when finding shared nibbles for our child nodes. (We know it's + // the first of the input keys, because the input is sorted) + // let mut begin = match value { + // None => 0, + // _ => 1, + // }; + // for i in 0..16 { + // shared_nibble_counts[i] = input[begin..] + // .iter() + // .take_while(|(k, _)| k.as_ref()[cursor] == i as u8) + // .count(); + // begin += shared_nibble_counts[i]; + // } + var begin uint + if val == nil { + begin = 0 + } else { + begin = 1 + } + for i := uint(0); i < 16; i++ { + var sharedNibbleCount uint + for _, kv := range input[begin:] { + if kv.Key[cursor] == byte(i) { + sharedNibbleCount++ + } else { + break + } + } + sharedNibblesCounts[i] = sharedNibbleCount + begin += sharedNibbleCount + } + } + + // Put out the node header: + // let value = value.map(|v| Value::new::(v, threshold)); + // stream.begin_branch( + // o_branch_slice, + // value.clone(), + // shared_nibble_counts.iter().map(|&n| n > 0), + // ); + var v Value + if val != nil { + v = NewValue[Hasher](val, threshold) + } + hasChildren := make([]bool, 16) + for i, count := range sharedNibblesCounts { + if count > 0 { + hasChildren[i] = true + } + } + stream.BeginBranch(oBranchSlice, v, hasChildren) + + // Fill in each slot in the branch node. We don't need to bother with empty slots + // since they were registered in the header. + // let mut begin = match value { + // None => 0, + // _ => 1, + // }; + var begin uint = 0 + if val != nil { + begin = 1 + } + // for &count in &shared_nibble_counts { + // if count > 0 { + // build_trie_trampoline::( + // &input[begin..(begin + count)], + // cursor + 1, + // stream, + // no_extension, + // threshold.clone(), + // ); + // begin += count; + // } else { + // stream.append_empty_child(); + // } + // } + for _, count := range sharedNibblesCounts { + if count > 0 { + buildTrieTrampoline[Hasher, H]( + input[begin:begin+count], + cursor+1, + stream, + threshold, + ) + begin += count + } else { + stream.AppendEmptyChild() + } + } + + // stream.end_branch(value); + stream.EndBranch(v) + } +} + +// fn build_trie_trampoline( +// +// input: &[(A, B)], +// cursor: usize, +// stream: &mut S, +// no_extension: bool, +// threshold: Option, +// +// ) where +// +// A: AsRef<[u8]>, +// B: AsRef<[u8]>, +// H: Hasher, +// S: TrieStream, +// +// { +func buildTrieTrampoline[Hasher hashdb.Hasher[H], H hashdb.Hash]( + input []KeyValue, + cursor uint, + stream TrieStream, + threshold *uint32, +) { + // let mut substream = S::new(); + substream := stream.New() + // build_trie::(input, cursor, &mut substream, no_extension, threshold); + buildTrie[Hasher, H](input, cursor, substream, threshold) + // stream.append_substream::(substream); + stream.AppendSubstream(substream) +} diff --git a/internal/primitives/trie/trie.go b/internal/primitives/trie/trie.go index 3bea2a31a7..91a1f7bd03 100644 --- a/internal/primitives/trie/trie.go +++ b/internal/primitives/trie/trie.go @@ -4,22 +4,52 @@ package trie import ( + "math" "slices" hashdb "github.com/ChainSafe/gossamer/internal/hash-db" memorydb "github.com/ChainSafe/gossamer/internal/memory-db" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/kv" + trieroot "github.com/ChainSafe/gossamer/internal/primitives/trie/trie-root" triedb "github.com/ChainSafe/gossamer/pkg/trie/triedb" ) +type Layout[H hashdb.Hash] interface { + MaxInlineValue() int + TrieRoot(input []kv.KeyValue) H +} + +type ( + // / substrate trie layout + LayoutV0[Hasher hashdb.Hasher[H], H hashdb.Hash] struct{} + // / substrate trie layout, with external value nodes. + LayoutV1[Hasher hashdb.Hasher[H], H hashdb.Hash] struct{} +) + +func (l LayoutV0[Hasher, H]) MaxInlineValue() int { + return math.MaxInt +} +func (l LayoutV1[Hasher, H]) MaxInlineValue() int { + return 32 +} + +func (l LayoutV0[Hasher, H]) TrieRoot(input []kv.KeyValue) H { + return trieroot.TrieRoot[Hasher, H](input, nil, NewTrieStream[Hasher, H]()) +} + +func (l LayoutV1[Hasher, H]) TrieRoot(input []kv.KeyValue) H { + max := uint32(32) + return trieroot.TrieRoot[Hasher, H](input, &max, NewTrieStream[Hasher, H]()) +} + // PrefixedMemoryDB is reexport from [memorydb.MemoryDB] where supplied [memorydb.KeyFunction] is [memorydb.PrefixedKey] // for prefixing keys internally (avoiding key conflict for non random keys). -type PrefixedMemoryDB[Hash runtime.Hash, Hasher hashdb.Hasher[Hash]] struct { +type PrefixedMemoryDB[Hash hashdb.Hash, Hasher hashdb.Hasher[Hash]] struct { memorydb.MemoryDB[Hash, Hasher, string, memorydb.PrefixedKey[Hash]] } // NewPrefixedMemoryDB is constructor for [PrefixedMemoryDB] -func NewPrefixedMemoryDB[Hash runtime.Hash, Hasher hashdb.Hasher[Hash]]() *PrefixedMemoryDB[Hash, Hasher] { +func NewPrefixedMemoryDB[Hash hashdb.Hash, Hasher hashdb.Hasher[Hash]]() *PrefixedMemoryDB[Hash, Hasher] { return &PrefixedMemoryDB[Hash, Hasher]{ memorydb.NewMemoryDB[Hash, Hasher, string, memorydb.PrefixedKey[Hash]]([]byte{0}), } @@ -27,25 +57,22 @@ func NewPrefixedMemoryDB[Hash runtime.Hash, Hasher hashdb.Hasher[Hash]]() *Prefi // MemoryDB is reexport from [memorydb.MemoryDB] where supplied [memorydb.KeyFunction] is [memorydb.HashKey] which is // a noop operation on the supplied prefix, and only uses the hash. -type MemoryDB[Hash runtime.Hash, Hasher runtime.Hasher[Hash]] struct { +type MemoryDB[Hash hashdb.Hash, Hasher hashdb.Hasher[Hash]] struct { memorydb.MemoryDB[Hash, Hasher, Hash, memorydb.HashKey[Hash]] } // NewMemoryDB is constructor for [MemoryDB]. -func NewMemoryDB[Hash runtime.Hash, Hasher runtime.Hasher[Hash]]() *MemoryDB[Hash, Hasher] { +func NewMemoryDB[Hash hashdb.Hash, Hasher hashdb.Hasher[Hash]]() *MemoryDB[Hash, Hasher] { return &MemoryDB[Hash, Hasher]{ MemoryDB: memorydb.NewMemoryDB[Hash, Hasher, Hash, memorydb.HashKey[Hash]]([]byte{0}), } } // KeyValue is a byte slice for key and value, where the value can be optional (nil). -type KeyValue struct { - Key []byte - Value []byte -} +type KeyValue = kv.KeyValue // DeltaTrieRoot determines a trie root given a hash DB and delta values. -func DeltaTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]( +func DeltaTrieRoot[H hashdb.Hash, Hasher hashdb.Hasher[H]]( db hashdb.HashDB[H], root H, delta []KeyValue, @@ -53,8 +80,7 @@ func DeltaTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]( cache triedb.TrieCache[H], stateVersion triedb.TrieLayout, ) (H, error) { - trieDB := triedb.NewTrieDB(root, db, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + trieDB := triedb.NewTrieDB(root, db, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) slices.SortStableFunc(delta, func(a KeyValue, b KeyValue) int { if string(a.Key) < string(b.Key) { @@ -86,7 +112,7 @@ func DeltaTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]( } // ReadTrieValue reads a value from the trie. -func ReadTrieValue[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ReadTrieValue[H hashdb.Hash, Hasher hashdb.Hasher[H]]( db hashdb.HashDB[H], root H, key []byte, @@ -94,8 +120,7 @@ func ReadTrieValue[H runtime.Hash, Hasher runtime.Hasher[H]]( cache triedb.TrieCache[H], stateVersion triedb.TrieLayout, ) ([]byte, error) { - trieDB := triedb.NewTrieDB(root, db, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + trieDB := triedb.NewTrieDB(root, db, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) b, err := triedb.GetWith(trieDB, key, func(data []byte) []byte { return data }) if err != nil { return nil, err @@ -107,7 +132,7 @@ func ReadTrieValue[H runtime.Hash, Hasher runtime.Hasher[H]]( } // ReadTrieValueWith reads a value from the trie with given [triedb.Query]. -func ReadTrieValueWith[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ReadTrieValueWith[H hashdb.Hash, Hasher hashdb.Hasher[H]]( db hashdb.HashDB[H], root H, key []byte, @@ -116,8 +141,7 @@ func ReadTrieValueWith[H runtime.Hash, Hasher runtime.Hasher[H]]( stateVersion triedb.TrieLayout, query triedb.Query[[]byte], ) ([]byte, error) { - trieDB := triedb.NewTrieDB(root, db, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + trieDB := triedb.NewTrieDB(root, db, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) b, err := triedb.GetWith(trieDB, key, query) if err != nil { return nil, err @@ -130,7 +154,7 @@ func ReadTrieValueWith[H runtime.Hash, Hasher runtime.Hasher[H]]( // ReadTrieFirstDescendantValue reads the [triedb.MerkleValue] of the node that is the closest descendant for // the provided key. -func ReadTrieFirstDescendantValue[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ReadTrieFirstDescendantValue[H hashdb.Hash, Hasher hashdb.Hasher[H]]( db hashdb.HashDB[H], root H, key []byte, @@ -138,26 +162,25 @@ func ReadTrieFirstDescendantValue[H runtime.Hash, Hasher runtime.Hasher[H]]( cache triedb.TrieCache[H], stateVersion triedb.TrieLayout, ) (triedb.MerkleValue[H], error) { - trieDB := triedb.NewTrieDB(root, db, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + trieDB := triedb.NewTrieDB(root, db, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) return trieDB.LookupFirstDescendant(key) } // EmptyTrieRoot returns the empty trie root. -func EmptyTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]() H { +func EmptyTrieRoot[H hashdb.Hash, Hasher hashdb.Hasher[H]]() H { hasher := *new(Hasher) root := hasher.Hash([]byte{0}) return root } // EmptyChildTrieRoot returns the empty child trie root. -func EmptyChildTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]() H { +func EmptyChildTrieRoot[H hashdb.Hash, Hasher hashdb.Hasher[H]]() H { return EmptyTrieRoot[H, Hasher]() } // ChildDeltaTrieRoot determines a child trie root given a hash DB and delta values. -func ChildDeltaTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ChildDeltaTrieRoot[H hashdb.Hash, Hasher hashdb.Hasher[H]]( keyspace []byte, db hashdb.HashDB[H], root H, @@ -171,7 +194,7 @@ func ChildDeltaTrieRoot[H runtime.Hash, Hasher runtime.Hasher[H]]( } // ReadChildTrieValue reads a value from the child trie. -func ReadChildTrieValue[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ReadChildTrieValue[H hashdb.Hash, Hasher hashdb.Hasher[H]]( keyspace []byte, db hashdb.HashDB[H], root H, @@ -182,8 +205,7 @@ func ReadChildTrieValue[H runtime.Hash, Hasher runtime.Hasher[H]]( ) ([]byte, error) { ksdb := NewKeyspacedDB(db, keyspace) trieDB := triedb.NewTrieDB( - root, ksdb, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + root, ksdb, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) val, err := triedb.GetWith(trieDB, key, func(data []byte) []byte { return data }) if err != nil { return nil, err @@ -195,7 +217,7 @@ func ReadChildTrieValue[H runtime.Hash, Hasher runtime.Hasher[H]]( } // ReadChildTrieHash reads a hash from the child trie. -func ReadChildTrieHash[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ReadChildTrieHash[H hashdb.Hash, Hasher hashdb.Hasher[H]]( keyspace []byte, db hashdb.HashDB[H], root H, @@ -206,14 +228,13 @@ func ReadChildTrieHash[H runtime.Hash, Hasher runtime.Hasher[H]]( ) (*H, error) { ksdb := NewKeyspacedDB(db, keyspace) trieDB := triedb.NewTrieDB( - root, ksdb, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + root, ksdb, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) return trieDB.GetHash(key) } // ReadChildTrieFirstDescendantValue reads the [triedb.MerkleValue] of the node that is the closest descendant for // the provided child key. -func ReadChildTrieFirstDescendantValue[H runtime.Hash, Hasher runtime.Hasher[H]]( +func ReadChildTrieFirstDescendantValue[H hashdb.Hash, Hasher hashdb.Hasher[H]]( keyspace []byte, db hashdb.HashDB[H], root H, @@ -224,8 +245,7 @@ func ReadChildTrieFirstDescendantValue[H runtime.Hash, Hasher runtime.Hasher[H]] ) (triedb.MerkleValue[H], error) { ksdb := NewKeyspacedDB(db, keyspace) trieDB := triedb.NewTrieDB( - root, ksdb, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) - trieDB.SetVersion(stateVersion) + root, ksdb, stateVersion, triedb.WithCache[H, Hasher](cache), triedb.WithRecorder[H, Hasher](recorder)) return trieDB.LookupFirstDescendant(key) } @@ -280,3 +300,26 @@ func (tbe *KeyspacedDB[H]) Remove(key H, prefix hashdb.Prefix) { derivedPrefix := keyspaceAsPrefix(tbe.keySpace, prefix) tbe.db.Remove(key, derivedPrefix) } + +// mod trie_constants { +const firstPrefix uint8 = 0b_00 << 6 + +const leafPrefixMask uint8 = 0b_01 << 6 + +// pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; +const branchWithoutMask uint8 = 0b_10 << 6 + +// pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; +const branchWithMask uint8 = 0b_11 << 6 +const emptyTrie uint8 = firstPrefix | (0b_00 << 4) + +// pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_1 << 5); +const altHashingLeafPrefixMask uint8 = firstPrefix | (0b_1 << 5) + +// pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); +const altHashingBranchWithMask uint8 = firstPrefix | (0b_01 << 4) + +// pub const ESCAPE_COMPACT_HEADER: u8 = EMPTY_TRIE | 0b_00_01; +const ESCAPE_COMPACT_HEADER uint8 = emptyTrie | 0b_00_01 + +// } diff --git a/internal/primitives/trie/trie_stream.go b/internal/primitives/trie/trie_stream.go new file mode 100644 index 0000000000..44ee0fd94e --- /dev/null +++ b/internal/primitives/trie/trie_stream.go @@ -0,0 +1,255 @@ +package trie + +import ( + hashdb "github.com/ChainSafe/gossamer/internal/hash-db" + trieroot "github.com/ChainSafe/gossamer/internal/primitives/trie/trie-root" + "github.com/ChainSafe/gossamer/pkg/scale" +) + +// / Codec-flavored TrieStream. +// #[derive(Default, Clone)] +// pub struct TrieStream { +type TrieStream[Hasher hashdb.Hasher[H], H hashdb.Hash] struct { + /// Current node buffer. + // buffer: Vec, + buffer []byte +} + +// impl TrieStream { +// // useful for debugging but not used otherwise +// pub fn as_raw(&self) -> &[u8] { +// &self.buffer +// } +// } + +// use trie_root::Value as TrieStreamValue; +// +// impl trie_root::TrieStream for TrieStream { +// fn new() -> Self { +// Self { buffer: Vec::new() } +// } +func (ts *TrieStream[Hasher, H]) New() trieroot.TrieStream { + return NewTrieStream[Hasher, H]() +} +func NewTrieStream[Hasher hashdb.Hasher[H], H hashdb.Hash]() *TrieStream[Hasher, H] { + return &TrieStream[Hasher, H]{buffer: make([]byte, 0)} +} + +// fn append_empty_data(&mut self) { +// self.buffer.push(trie_constants::EMPTY_TRIE); +// } +func (ts *TrieStream[Hasher, H]) AppendEmptyData() { + ts.buffer = append(ts.buffer, emptyTrie) +} + +// fn append_leaf(&mut self, key: &[u8], value: TrieStreamValue) { +// let kind = match &value { +// TrieStreamValue::Inline(..) => NodeKind::Leaf, +// TrieStreamValue::Node(..) => NodeKind::HashedValueLeaf, +// }; +// self.buffer.extend(fuse_nibbles_node(key, kind)); +// match &value { +// TrieStreamValue::Inline(value) => { +// Compact(value.len() as u32).encode_to(&mut self.buffer); +// self.buffer.extend_from_slice(value); +// }, +// TrieStreamValue::Node(hash) => { +// self.buffer.extend_from_slice(hash.as_slice()); +// }, +// }; +// } +func (ts *TrieStream[Hasher, H]) AppendLeaf(key []byte, value trieroot.Value) { + var kind nodeKind + switch value.(type) { + case trieroot.InlineValue: + kind = nodeKindLeaf + case trieroot.NodeValue: + kind = nodeKindHashedValueLeaf + default: + panic("unreachable") + } + ts.buffer = append(ts.buffer, fuseNibblesNode(key, kind)...) + switch value := value.(type) { + case trieroot.InlineValue: + length := scale.MustMarshal(len(value)) + ts.buffer = append(ts.buffer, length...) + ts.buffer = append(ts.buffer, value...) + case trieroot.NodeValue: + ts.buffer = append(ts.buffer, value...) + default: + panic("unreachable") + } +} + +// fn begin_branch( +// &mut self, +// maybe_partial: Option<&[u8]>, +// maybe_value: Option, +// has_children: impl Iterator, +// ) { +// if let Some(partial) = maybe_partial { +// let kind = match &maybe_value { +// None => NodeKind::BranchNoValue, +// Some(TrieStreamValue::Inline(..)) => NodeKind::BranchWithValue, +// Some(TrieStreamValue::Node(..)) => NodeKind::HashedValueBranch, +// }; + +// self.buffer.extend(fuse_nibbles_node(partial, kind)); +// let bm = branch_node_bit_mask(has_children); +// self.buffer.extend([bm.0, bm.1].iter()); +// } else { +// unreachable!("trie stream codec only for no extension trie"); +// } +// match maybe_value { +// None => (), +// Some(TrieStreamValue::Inline(value)) => { +// Compact(value.len() as u32).encode_to(&mut self.buffer); +// self.buffer.extend_from_slice(value); +// }, +// Some(TrieStreamValue::Node(hash)) => { +// self.buffer.extend_from_slice(hash.as_slice()); +// }, +// } +// } +func (ts *TrieStream[Hasher, H]) BeginBranch(maybePartial []byte, maybeValue trieroot.Value, hasChildren []bool) { + if maybePartial != nil { + var kind nodeKind + if maybeValue == nil { + kind = nodeKindBranchNoValue + } else { + switch maybeValue.(type) { + case trieroot.InlineValue: + kind = nodeKindBranchWithValue + case trieroot.NodeValue: + kind = nodeKindHashedValueBranch + default: + panic("unreachable") + } + } + + ts.buffer = append(ts.buffer, fuseNibblesNode(maybePartial, kind)...) + a, b := branchNodeBitMask(hasChildren) + ts.buffer = append(ts.buffer, a, b) + } else { + panic("trie stream codec only for no extension trie") + } + if maybeValue == nil { + return + } + switch value := maybeValue.(type) { + case trieroot.InlineValue: + length := scale.MustMarshal(len(value)) + ts.buffer = append(ts.buffer, length...) + ts.buffer = append(ts.buffer, value...) + case trieroot.NodeValue: + ts.buffer = append(ts.buffer, value...) + default: + panic("unreachable") + } +} + +// fn append_extension(&mut self, _key: &[u8]) { +// debug_assert!(false, "trie stream codec only for no extension trie"); +// } +func (ts *TrieStream[Hasher, H]) AppendExtension(key []byte) { + panic("trie stream codec only for no extension trie") +} + +// fn append_substream(&mut self, other: Self) { +// let data = other.out(); +// match data.len() { +// 0..=31 => data.encode_to(&mut self.buffer), +// _ => H::hash(&data).as_ref().encode_to(&mut self.buffer), +// } +// } +func (ts *TrieStream[Hasher, H]) AppendSubstream(other trieroot.TrieStream) { + data := other.Out() + if len(data) <= 31 { + ts.buffer = append(ts.buffer, scale.MustMarshal(data)...) + } else { + hash := (*new(Hasher)).Hash(data) + ts.buffer = append(ts.buffer, scale.MustMarshal(hash.Bytes())...) + } +} + +// fn out(self) -> Vec { +// self.buffer +// } +// } +func (ts *TrieStream[Hasher, H]) Out() []byte { + return ts.buffer +} + +func (ts *TrieStream[Hasher, H]) AppendEmptyChild() {} +func (ts *TrieStream[Hasher, H]) EndBranch(value trieroot.Value) {} + +// fn branch_node_bit_mask(has_children: impl Iterator) -> (u8, u8) { +func branchNodeBitMask(hasChildren []bool) (uint8, uint8) { + // let mut bitmap: u16 = 0; + // let mut cursor: u16 = 1; + var ( + bitmap uint16 + cursor uint16 = 1 + ) + // + // for v in has_children { + // if v { + // bitmap |= cursor + // } + // cursor <<= 1; + // } + for _, v := range hasChildren { + if v { + bitmap |= cursor + } + cursor <<= 1 + } + // + // ((bitmap % 256) as u8, (bitmap / 256) as u8) + return uint8(bitmap % 256), uint8(bitmap / 256) +} + +// /// Create a leaf/branch node, encoding a number of nibbles. +// fn fuse_nibbles_node(nibbles: &[u8], kind: NodeKind) -> impl Iterator + '_ { +func fuseNibblesNode(nibbles []byte, kind nodeKind) []byte { + // let size = nibbles.len(); + size := uint(len(nibbles)) + // + // let iter_start = match kind { + // NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), + // NodeKind::BranchNoValue => + // size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), + // NodeKind::BranchWithValue => + // size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), + // NodeKind::HashedValueLeaf => + // size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), + // NodeKind::HashedValueBranch => + // size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), + // }; + var start []byte + switch kind { + case nodeKindLeaf: + start = sizeAndPrefixIterator(size, leafPrefixMask, 2) + case nodeKindBranchNoValue: + start = sizeAndPrefixIterator(size, branchWithoutMask, 2) + case nodeKindBranchWithValue: + start = sizeAndPrefixIterator(size, branchWithMask, 2) + case nodeKindHashedValueLeaf: + start = sizeAndPrefixIterator(size, altHashingLeafPrefixMask, 3) + case nodeKindHashedValueBranch: + start = sizeAndPrefixIterator(size, altHashingBranchWithMask, 4) + } + // + // iter_start + // + // .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) + // .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) + if len(nibbles)%2 == 1 { + start = append(start, nibbles[0]) + } + begin := len(nibbles) % 2 + for i := begin; i < len(nibbles); i += 2 { + start = append(start, nibbles[i]<<4|nibbles[i+1]) + } + return start +} diff --git a/internal/primitives/trie/trie_test.go b/internal/primitives/trie/trie_test.go index f3f95f493d..2d2647b22d 100644 --- a/internal/primitives/trie/trie_test.go +++ b/internal/primitives/trie/trie_test.go @@ -4,10 +4,241 @@ package trie import ( + "testing" + hashdb "github.com/ChainSafe/gossamer/internal/hash-db" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" + "github.com/ChainSafe/gossamer/pkg/trie/triedb" + "github.com/stretchr/testify/require" ) var ( _ hashdb.HashDB[hash.H256] = &KeyspacedDB[hash.H256]{} ) + +// fn check_equivalent(input: &Vec<(&[u8], &[u8])>) { +func checkEquivalent(t *testing.T, input []KeyValue, layout Layout[hash.H256]) { + + // { + // let closed_form = T::trie_root(input.clone()); + // let d = T::trie_root_unhashed(input.clone()); + // println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); + // let persistent = { + // let mut memdb = MemoryDBMeta::default(); + // let mut root = Default::default(); + // let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); + // for (x, y) in input.iter().rev() { + // t.insert(x, y).unwrap(); + // } + // *t.root() + // }; + // assert_eq!(closed_form, persistent); + // } + closedForm := layout.TrieRoot(input) + + t.Logf("%s", closedForm) + + memDB := NewMemoryDB[hash.H256, hasher.Blake2Hasher]() + trieDB := triedb.NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](memDB, layout) + for i := len(input) - 1; i >= 0; i-- { + kv := input[i] + trieDB.Set(kv.Key, kv.Value) + } + persistent := trieDB.MustHash() + require.Equal(t, closedForm, persistent) +} + +// fn check_iteration(input: &Vec<(&[u8], &[u8])>) { +// let mut memdb = MemoryDBMeta::default(); +// let mut root = Default::default(); +// { +// let mut t = TrieDBMutBuilder::::new(&mut memdb, &mut root).build(); +// for (x, y) in input.clone() { +// t.insert(x, y).unwrap(); +// } +// } +// { +// let t = TrieDBBuilder::::new(&memdb, &root).build(); +// assert_eq!( +// input.iter().map(|(i, j)| (i.to_vec(), j.to_vec())).collect::>(), +// t.iter() +// .unwrap() +// .map(|x| x.map(|y| (y.0, y.1.to_vec())).unwrap()) +// .collect::>() +// ); +// } +// } + +// fn check_input(input: &Vec<(&[u8], &[u8])>) { +// check_equivalent::(input); +// check_iteration::(input); +// check_equivalent::(input); +// check_iteration::(input); +// } +func checkInput(t *testing.T, input []KeyValue) { + checkEquivalent(t, input, LayoutV0[hasher.Blake2Hasher, hash.H256]{}) + // check_iteration::(input) + checkEquivalent(t, input, LayoutV1[hasher.Blake2Hasher, hash.H256]{}) + // check_iteration::(input) +} + +func TestTrieRoot(t *testing.T) { + // fn default_trie_root() { + t.Run("default_trie_root", func(t *testing.T) { + // let mut db = MemoryDB::default(); + db := NewMemoryDB[hash.H256, hasher.Blake2Hasher]() + // let mut root = TrieHash::::default(); + // root := hash.NewH256() + // let mut empty = TrieDBMutBuilder::::new(&mut db, &mut root).build(); + empty := triedb.NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, LayoutV1[hasher.Blake2Hasher, hash.H256]{}) + // empty.commit(); + root1 := empty.MustHash() + // let root1 = empty.root().as_ref().to_vec(); + // let root2: Vec = LayoutV1::trie_root::<_, Vec, Vec>(std::iter::empty()) + // .as_ref() + // .iter() + // .cloned() + // .collect(); + root2 := LayoutV1[hasher.Blake2Hasher, hash.H256]{}.TrieRoot(nil) + require.Equal(t, root1, root2) + // assert_eq!(root1, root2); + }) + + // fn empty_is_equivalent() { + t.Run("empty_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = vec![]; + // check_input(&input); + checkInput(t, nil) + }) + + // fn leaf_is_equivalent() { + t.Run("leaf_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0xbb][..])]; + // check_input(&input); + checkInput(t, []KeyValue{ + {Key: []byte{0xaa}, Value: []byte{0xbb}}, + }) + }) + + // #[test] + // fn branch_is_equivalent() { + t.Run("branch_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = + // vec![(&[0xaa][..], &[0x10][..]), (&[0xba][..], &[0x11][..])]; + input := []KeyValue{ + {Key: []byte{0xaa}, Value: []byte{0x10}}, + {Key: []byte{0xba}, Value: []byte{0x11}}, + } + // check_input(&input); + checkInput(t, input) + }) + + // #[test] + // fn extension_and_branch_is_equivalent() { + t.Run("extension_and_branch_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = + // vec![(&[0xaa][..], &[0x10][..]), (&[0xab][..], &[0x11][..])]; + // check_input(&input); + input := []KeyValue{ + {Key: []byte{0xaa}, Value: []byte{0x10}}, + {Key: []byte{0xab}, Value: []byte{0x11}}, + } + checkInput(t, input) + }) + + // #[test] + // fn standard_is_equivalent() { + // let st = StandardMap { + // alphabet: Alphabet::All, + // min_key: 32, + // journal_key: 0, + // value_mode: ValueMode::Random, + // count: 1000, + // }; + // let mut d = st.make(); + // d.sort_by(|(a, _), (b, _)| a.cmp(b)); + // let dr = d.iter().map(|v| (&v.0[..], &v.1[..])).collect(); + // check_input(&dr); + // } + + // #[test] + // fn extension_and_branch_with_value_is_equivalent() { + t.Run("extension_and_branch_with_value_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = vec![ + // (&[0xaa][..], &[0xa0][..]), + // (&[0xaa, 0xaa][..], &[0xaa][..]), + // (&[0xaa, 0xbb][..], &[0xab][..]), + // ]; + // check_input(&input); + input := []KeyValue{ + {Key: []byte{0xaa}, Value: []byte{0xa0}}, + {Key: []byte{0xaa, 0xaa}, Value: []byte{0xaa}}, + {Key: []byte{0xaa, 0xbb}, Value: []byte{0xab}}, + } + checkInput(t, input) + }) + + // #[test] + // fn bigger_extension_and_branch_with_value_is_equivalent() { + t.Run("bigger_extension_and_branch_with_value_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = vec![ + // (&[0xaa][..], &[0xa0][..]), + // (&[0xaa, 0xaa][..], &[0xaa][..]), + // (&[0xaa, 0xbb][..], &[0xab][..]), + // (&[0xbb][..], &[0xb0][..]), + // (&[0xbb, 0xbb][..], &[0xbb][..]), + // (&[0xbb, 0xcc][..], &[0xbc][..]), + // ]; + // check_input(&input); + input := []KeyValue{ + {Key: []byte{0xaa}, Value: []byte{0xa0}}, + {Key: []byte{0xaa, 0xaa}, Value: []byte{0xaa}}, + {Key: []byte{0xaa, 0xbb}, Value: []byte{0xab}}, + {Key: []byte{0xbb}, Value: []byte{0xb0}}, + {Key: []byte{0xbb, 0xbb}, Value: []byte{0xbb}}, + {Key: []byte{0xbb, 0xcc}, Value: []byte{0xbc}}, + } + checkInput(t, input) + }) + + // #[test] + // fn single_long_leaf_is_equivalent() { + t.Run("single_long_leaf_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = vec![ + // ( + // &[0xaa][..], + // &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..], + // ), + // (&[0xba][..], &[0x11][..]), + // ]; + // check_input(&input); + input := []KeyValue{ + {Key: []byte{0xaa}, Value: []byte("ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC")}, + {Key: []byte{0xba}, Value: []byte{0x11}}, + } + checkInput(t, input) + }) + + // #[test] + // fn two_long_leaves_is_equivalent() { + t.Run("two_long_leaves_is_equivalent", func(t *testing.T) { + // let input: Vec<(&[u8], &[u8])> = vec![ + // ( + // &[0xaa][..], + // &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..], + // ), + // ( + // &[0xba][..], + // &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..], + // ), + // ]; + // check_input(&input); + input := []KeyValue{ + {Key: []byte{0xaa}, Value: []byte("ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC")}, + {Key: []byte{0xba}, Value: []byte("ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC")}, + } + checkInput(t, input) + }) + +} diff --git a/internal/primitives/version/version.go b/internal/primitives/version/version.go index 88b20bc40d..5fca2b9c27 100644 --- a/internal/primitives/version/version.go +++ b/internal/primitives/version/version.go @@ -16,13 +16,10 @@ type ApiID [8]uint8 // A pair of ApiID and a uint32 for version. type ApiIDVersion struct { - ApiId ApiID + ApiID Version uint32 } -// A vector of pairs of ApiID and a uint32 for version. -type ApisVec []ApiIDVersion - // Runtime version. // This should not be thought of as classic Semver (major/minor/tiny). // This triplet have different semantics and mis-interpretation could cause problems. @@ -67,7 +64,7 @@ type RuntimeVersion struct { ImplVersion uint32 // List of supported API "features" along with their versions. - Apis ApisVec + APIs []ApiIDVersion // All existing calls (dispatchables) are fully compatible when this number doesn't change. If // this number changes, then SpecVersion must change, also. @@ -89,3 +86,17 @@ func (v *RuntimeVersion) StateVersion() storage.StateVersion { return storage.StateVersion(stateVersion) } + +// / Returns the api version found for api with `id`. +// +// pub fn api_version(&self, id: &ApiId) -> Option { +// self.apis.iter().find_map(|a| (a.0 == *id).then(|| a.1)) +// } +func (v *RuntimeVersion) APIVersion(id ApiID) *uint32 { + for _, api := range v.APIs { + if api.ApiID == id { + return &api.Version + } + } + return nil +} diff --git a/internal/test-utils/client/client.go b/internal/test-utils/client/client.go new file mode 100644 index 0000000000..bc41fc1907 --- /dev/null +++ b/internal/test-utils/client/client.go @@ -0,0 +1,401 @@ +package client + +import ( + "math" + + "github.com/ChainSafe/gossamer/internal/client" + "github.com/ChainSafe/gossamer/internal/client/api" + chainspec "github.com/ChainSafe/gossamer/internal/client/chain-spec" + "github.com/ChainSafe/gossamer/internal/client/consensus/common" + "github.com/ChainSafe/gossamer/internal/client/db" + "github.com/ChainSafe/gossamer/internal/client/executor" + "github.com/ChainSafe/gossamer/internal/primitives/core" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/storage" + "github.com/tidwall/btree" +) + +// / A genesis storage initialization trait. +// pub trait GenesisInit: Default { +type GenesisInit interface { + /// Construct genesis storage. + // fn genesis_storage(&self) -> Storage; + GenesisStorage() storage.Storage +} + +type DefaultGenesisInit struct{} + +func (d DefaultGenesisInit) GenesisStorage() storage.Storage { + return storage.Storage{ + Top: btree.Map[string, []byte]{}, + ChildrenDefault: make(map[string]storage.StorageChild), + } +} + +// / A builder for creating a test client instance. +// pub struct TestClientBuilder { +type TestClientBuilder[ + H runtime.Hash, + Hasher runtime.Hasher[H], + N runtime.Number, + E runtime.Extrinsic, + Header runtime.Header[N, H], + G GenesisInit, +] struct { + // genesis_init: G, + genesisInit G + /// The key is an unprefixed storage key, this only contains + /// default child trie content. + // child_storage_extension: HashMap, StorageChild>, + childStorageExtension map[string]storage.StorageChild + // backend: Arc, + backend *db.Backend[H, Hasher, N, E, Header] + // _executor: std::marker::PhantomData, + // fork_blocks: ForkBlocks, + forkBlocks api.ForkBlocks[H, N] + // bad_blocks: BadBlocks, + badBlocks api.BadBlocks[H] + // enable_offchain_indexing_api: bool, + enableOffchainIndexingAPI bool + // enable_import_proof_recording: bool, + enableImportProofRecording bool + // no_genesis: bool, + noGenesis bool +} + +// impl Default +// for TestClientBuilder, G> +// { +// fn default() -> Self { +// Self::with_default_backend() +// } +// } + +// impl +// +// TestClientBuilder, G> +// +// { +// /// Create new `TestClientBuilder` with default backend. +// pub fn with_default_backend() -> Self { +// let backend = Arc::new(Backend::new_test(std::u32::MAX, std::u64::MAX)); +// Self::with_backend(backend) +// } +// +// / Create new `TestClientBuilder` with default backend. +func NewTestClientBuilderWithDefaultBackend[ + H runtime.Hash, + Hasher runtime.Hasher[H], + N runtime.Number, + E runtime.Extrinsic, + Header runtime.Header[N, H], + G GenesisInit, + +]() TestClientBuilder[H, Hasher, N, E, Header, G] { + backend := db.NewTestBackend[H, N, E, Hasher, Header](math.MaxUint32, math.MaxUint64) + return NewTestClientBuilderWithBackend[H, Hasher, N, E, Header, G](backend) +} + +// / Create new `TestClientBuilder` with default backend and pruning window size +// +// pub fn with_pruning_window(blocks_pruning: u32) -> Self { +// let backend = Arc::new(Backend::new_test(blocks_pruning, 0)); +// Self::with_backend(backend) +// } +func NewTestClientBuilderWithPruningWindow[ + H runtime.Hash, + Hasher runtime.Hasher[H], + N runtime.Number, + E runtime.Extrinsic, + Header runtime.Header[N, H], + G GenesisInit, +](blocksPruning uint32) TestClientBuilder[H, Hasher, N, E, Header, G] { + backend := db.NewTestBackend[H, N, E, Hasher, Header](blocksPruning, 0) + return NewTestClientBuilderWithBackend[H, Hasher, N, E, Header, G](backend) +} + +// / Create new `TestClientBuilder` with default backend and storage chain mode +// +// pub fn with_tx_storage(blocks_pruning: u32) -> Self { +// let backend = +// Arc::new(Backend::new_test_with_tx_storage(BlocksPruning::Some(blocks_pruning), 0)); +// Self::with_backend(backend) +// } +// } +func NewTestClientBuilderWithTxStorage[ + H runtime.Hash, + Hasher runtime.Hasher[H], + N runtime.Number, + E runtime.Extrinsic, + Header runtime.Header[N, H], + G GenesisInit, +](blocksPruning uint32) TestClientBuilder[H, Hasher, N, E, Header, G] { + backend := db.NewTestBackendWithTxStorage[H, N, E, Hasher, Header](db.BlocksPruningSome(blocksPruning), 0) + return NewTestClientBuilderWithBackend[H, Hasher, N, E, Header, G](backend) +} + +// impl +// +// TestClientBuilder +// +// { +// /// Create a new instance of the test client builder. +// pub fn with_backend(backend: Arc) -> Self { +// TestClientBuilder { +// backend, +// child_storage_extension: Default::default(), +// genesis_init: Default::default(), +// _executor: Default::default(), +// fork_blocks: None, +// bad_blocks: None, +// enable_offchain_indexing_api: false, +// no_genesis: false, +// enable_import_proof_recording: false, +// } +// } +func NewTestClientBuilderWithBackend[ + H runtime.Hash, + Hasher runtime.Hasher[H], + N runtime.Number, + E runtime.Extrinsic, + Header runtime.Header[N, H], + G GenesisInit, +](backend *db.Backend[H, Hasher, N, E, Header]) TestClientBuilder[H, Hasher, N, E, Header, G] { + return TestClientBuilder[H, Hasher, N, E, Header, G]{ + backend: backend, + childStorageExtension: make(map[string]storage.StorageChild), + // genesisInit: nil, + forkBlocks: nil, + badBlocks: nil, + enableOffchainIndexingAPI: false, + noGenesis: false, + enableImportProofRecording: false, + } +} + +// /// Alter the genesis storage parameters. +// +// pub fn genesis_init_mut(&mut self) -> &mut G { +// &mut self.genesis_init +// } +func (tcb *TestClientBuilder[H, Hasher, N, E, Header, G]) GenesisInitMut() G { + return tcb.genesisInit +} + +// /// Give access to the underlying backend of these clients +// +// pub fn backend(&self) -> Arc { +// self.backend.clone() +// } +func (tcb *TestClientBuilder[H, Hasher, N, E, Header, G]) Backend() *db.Backend[H, Hasher, N, E, Header] { + return tcb.backend +} + +// /// Extend child storage +// pub fn add_child_storage( +// mut self, +// child_info: &ChildInfo, +// key: impl AsRef<[u8]>, +// value: impl AsRef<[u8]>, +// ) -> Self { +// let storage_key = child_info.storage_key(); +// let entry = self.child_storage_extension.entry(storage_key.to_vec()).or_insert_with(|| { +// StorageChild { data: Default::default(), child_info: child_info.clone() } +// }); +// entry.data.insert(key.as_ref().to_vec(), value.as_ref().to_vec()); +// self +// } + +// /// Sets custom block rules. +// pub fn set_block_rules( +// mut self, +// fork_blocks: ForkBlocks, +// bad_blocks: BadBlocks, +// ) -> Self { +// self.fork_blocks = fork_blocks; +// self.bad_blocks = bad_blocks; +// self +// } + +// /// Enable the offchain indexing api. +// pub fn enable_offchain_indexing_api(mut self) -> Self { +// self.enable_offchain_indexing_api = true; +// self +// } + +// /// Enable proof recording on import. +// pub fn enable_import_proof_recording(mut self) -> Self { +// self.enable_import_proof_recording = true; +// self +// } + +// / Disable writing genesis. +// +// pub fn set_no_genesis(mut self) -> Self { +// self.no_genesis = true; +// self +// } +func (tcb *TestClientBuilder[H, Hasher, N, E, Header, G]) SetNoGenesis() { + tcb.noGenesis = true +} + +type ExecutorDispatch interface { + api.CallExecutor + core.CodeExecutor + executor.RuntimeVersionOf +} + +// /// Build the test client with the given native executor. +// pub fn build_with_executor( +// +// self, +// executor: ExecutorDispatch, +// +// ) -> ( +// +// client::Client, +// sc_consensus::LongestChain, +// +// ) +// where +// +// ExecutorDispatch: +// sc_client_api::CallExecutor + sc_executor::RuntimeVersionOf + Clone + 'static, +// Backend: sc_client_api::backend::Backend, +// >::OffchainStorage: 'static, +// +// { +func (tcb *TestClientBuilder[H, Hasher, N, E, Header, G]) BuildWithExecutor( + executor ExecutorDispatch, +) (*client.Client[H, Hasher, N, E, Header], common.LongestChain[H, N, Hasher, Header, E]) { + // let storage = { + // let mut storage = self.genesis_init.genesis_storage(); + // // Add some child storage keys. + // for (key, child_content) in self.child_storage_extension { + // storage.children_default.insert( + // key, + // StorageChild { + // data: child_content.data.into_iter().collect(), + // child_info: child_content.child_info, + // }, + // ); + // } + + // storage + // }; + store := tcb.genesisInit.GenesisStorage() + // Add some child storage keys. + for key, childContent := range tcb.childStorageExtension { + store.ChildrenDefault[key] = storage.StorageChild{ + Data: childContent.Data, + ChildInfo: childContent.ChildInfo, + } + } + + // let client_config = ClientConfig { + // enable_import_proof_recording: self.enable_import_proof_recording, + // offchain_indexing_api: self.enable_offchain_indexing_api, + // no_genesis: self.no_genesis, + // ..Default::default() + // }; + clientConfig := client.NewClientConfig[N]() + clientConfig.EnableImportProofRecording = tcb.enableImportProofRecording + clientConfig.OffchainIndexingAPI = tcb.enableOffchainIndexingAPI + clientConfig.NoGenesis = tcb.noGenesis + + // let genesis_block_builder = sc_service::GenesisBlockBuilder::new( + // &storage, + // !client_config.no_genesis, + // self.backend.clone(), + // executor.clone(), + // ) + // .expect("Creates genesis block builder"); + genesisBlockBuilder := chainspec.NewGenesisBlockBuilderWithStorage[H, N, Hasher, Header, E]( + store, + !clientConfig.NoGenesis, + tcb.backend, + executor, + ) + _ = genesisBlockBuilder + + // let spawn_handle = Box::new(TaskExecutor::new()); + + // let client = client::Client::new( + // self.backend.clone(), + // executor, + // spawn_handle, + // genesis_block_builder, + // self.fork_blocks, + // self.bad_blocks, + // None, + // None, + // client_config, + // ) + // .expect("Creates new client"); + client, err := client.New[H, Hasher, N, E, Header]( + tcb.backend, + clientConfig, + executor, + nil, + genesisBlockBuilder, + tcb.forkBlocks, + tcb.badBlocks, + ) + if err != nil { + panic("Creates new client") + } + + _ = client + // let longest_chain = sc_consensus::LongestChain::new(self.backend); + longestChain := common.NewLongestChain[H, N, Hasher, Header, E](tcb.backend) + + // (client, longest_chain) + return client, longestChain +} + +// } + +// impl +// +// TestClientBuilder>, Backend, G> +// +// { +// /// Build the test client with the given native executor. +// pub fn build_with_native_executor( +// self, +// executor: I, +// ) -> ( +// client::Client< +// Backend, +// client::LocalCallExecutor>, +// Block, +// RuntimeApi, +// >, +// sc_consensus::LongestChain, +// ) +// where +// I: Into>>, +// Backend: sc_client_api::backend::Backend + 'static, +// H: sc_executor::HostFunctions, +// { +func (tcb *TestClientBuilder[H, Hasher, N, E, Header, G]) BuildWithNativeExecutor( + exec *executor.WasmExecutor, +) (*client.Client[H, Hasher, N, E, Header], common.LongestChain[H, N, Hasher, Header, E]) { + + // let executor = executor.into().unwrap_or_else(|| WasmExecutor::::builder().build()); + // let executor = LocalCallExecutor::new( + // self.backend.clone(), + // executor.clone(), + // Default::default(), + // ExecutionExtensions::new(None, Arc::new(executor)), + // ) + // .expect("Creates LocalCallExecutor"); + if exec == nil { + exec = &executor.WasmExecutor{} + } + + // self.build_with_executor(executor) + return tcb.BuildWithExecutor(exec) +} + +// } diff --git a/internal/test-utils/runtime/client/client.go b/internal/test-utils/runtime/client/client.go new file mode 100644 index 0000000000..b25529c9b6 --- /dev/null +++ b/internal/test-utils/runtime/client/client.go @@ -0,0 +1,214 @@ +package client + +import ( + service_client "github.com/ChainSafe/gossamer/internal/client" + "github.com/ChainSafe/gossamer/internal/client/consensus/common" + "github.com/ChainSafe/gossamer/internal/client/db" + pruntime "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/storage" + "github.com/ChainSafe/gossamer/internal/test-utils/client" + "github.com/ChainSafe/gossamer/internal/test-utils/runtime" +) + +// / Test client database backend. +// pub type Backend = substrate_test_client::Backend; +type Backend = db.Backend[runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, runtime.Header] + +// / Parameters of test-client builder with test-runtime. +// #[derive(Default)] +// +// pub struct GenesisParameters { +// heap_pages_override: Option, +// extra_storage: Storage, +// wasm_code: Option>, +// } +type GenesisParameters struct { + heapPagesOverride *uint64 + extraStorage storage.Storage + wasmCode []byte +} + +// impl GenesisParameters { +// /// Set the wasm code that should be used at genesis. +// pub fn set_wasm_code(&mut self, code: Vec) { +// self.wasm_code = Some(code); +// } + +// /// Access extra genesis storage. +// pub fn extra_storage(&mut self) -> &mut Storage { +// &mut self.extra_storage +// } +// } +func (g *GenesisParameters) ExtraStorage() *storage.Storage { + return &g.extraStorage +} + +// impl GenesisInit for GenesisParameters { +// fn genesis_storage(&self) -> Storage { +// GenesisStorageBuilder::default() +// .with_heap_pages(self.heap_pages_override) +// .with_wasm_code(&self.wasm_code) +// .with_extra_storage(self.extra_storage.clone()) +// .build() +// } +// } +func (g GenesisParameters) GenesisStorage() storage.Storage { + panic("unimpl") +} + +type TestClientBuilder struct { + client.TestClientBuilder[ + runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, runtime.Header, *GenesisParameters, + ] +} + +func NewTestClientBuilderWithDefaultBackend() TestClientBuilder { + return TestClientBuilder{ + client.NewTestClientBuilderWithDefaultBackend[ + runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, runtime.Header, *GenesisParameters, + ](), + } +} + +// / Create new `TestClientBuilder` with default backend and pruning window size +// +// pub fn with_pruning_window(blocks_pruning: u32) -> Self { +// let backend = Arc::new(Backend::new_test(blocks_pruning, 0)); +// Self::with_backend(backend) +// } +func NewTestClientBuilderWithPruningWindow(blocksPruning uint32) TestClientBuilder { + return TestClientBuilder{ + client.NewTestClientBuilderWithPruningWindow[ + runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, runtime.Header, *GenesisParameters, + ](blocksPruning), + } +} + +// / Create new `TestClientBuilder` with default backend and storage chain mode +// +// pub fn with_tx_storage(blocks_pruning: u32) -> Self { +// let backend = +// Arc::new(Backend::new_test_with_tx_storage(BlocksPruning::Some(blocks_pruning), 0)); +// Self::with_backend(backend) +// } +// } +func NewTestClientBuilderWithTxStorage(blocksPruning uint32) TestClientBuilder { + return TestClientBuilder{ + client.NewTestClientBuilderWithTxStorage[ + runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, runtime.Header, *GenesisParameters, + ](blocksPruning), + } +} + +// / Test client type with `WasmExecutor` and generic Backend. +// pub type Client = client::Client< +// +// B, +// client::LocalCallExecutor, +// substrate_test_runtime::Block, +// substrate_test_runtime::RuntimeApi, +// +// >; +type Client = service_client.Client[ + runtime.Hash, runtime.Hasher, runtime.BlockNumber, runtime.Extrinsic, + runtime.Header, +] + +// / A `test-runtime` extensions to `TestClientBuilder`. +// pub trait TestClientBuilderExt: Sized { +type TestClientBuilderExt[ + H pruntime.Hash, + N pruntime.Number, + Hasher pruntime.Hasher[H], + Header pruntime.Header[N, H], + E pruntime.Extrinsic, +] interface { + // /// Returns a mutable reference to the genesis parameters. + // fn genesis_init_mut(&mut self) -> &mut GenesisParameters; + + // /// Override the default value for Wasm heap pages. + // fn set_heap_pages(mut self, heap_pages: u64) -> Self { + // self.genesis_init_mut().heap_pages_override = Some(heap_pages); + // self + // } + + // /// Add an extra value into the genesis storage. + // /// + // /// # Panics + // /// + // /// Panics if the key is empty. + // fn add_extra_child_storage>, V: Into>>( + // mut self, + // child_info: &ChildInfo, + // key: K, + // value: V, + // ) -> Self { + // let storage_key = child_info.storage_key().to_vec(); + // let key = key.into(); + // assert!(!storage_key.is_empty()); + // assert!(!key.is_empty()); + // self.genesis_init_mut() + // .extra_storage + // .children_default + // .entry(storage_key) + // .or_insert_with(|| StorageChild { + // data: Default::default(), + // child_info: child_info.clone(), + // }) + // .data + // .insert(key, value.into()); + // self + // } + + // /// Add an extra child value into the genesis storage. + // /// + // /// # Panics + // /// + // /// Panics if the key is empty. + // fn add_extra_storage>, V: Into>>(mut self, key: K, value: V) -> Self { + // let key = key.into(); + // assert!(!key.is_empty()); + // self.genesis_init_mut().extra_storage.top.insert(key, value.into()); + // self + // } + + // /// Build the test client. + // fn build(self) -> Client { + // self.build_with_longest_chain().0 + // } + + /// Build the test client and longest chain selector. + // fn build_with_longest_chain( + // self, + // ) -> (Client, sc_consensus::LongestChain); + BuildWithLongestChain() (Client, common.LongestChain[H, N, Hasher, Header, E]) + // /// Build the test client and the backend. + // fn build_with_backend(self) -> (Client, Arc); +} + +// impl TestClientBuilderExt +// for TestClientBuilder, B> +// where +// B: sc_client_api::backend::Backend + 'static, +// { +// fn genesis_init_mut(&mut self) -> &mut GenesisParameters { +// Self::genesis_init_mut(self) +// } + +// fn build_with_longest_chain( +// +// self, +// +// ) -> (Client, sc_consensus::LongestChain) { +// self.build_with_native_executor(None) +// } +func (tcb *TestClientBuilder) BuildWithLongestChain() (*Client, common.LongestChain[runtime.Hash, runtime.BlockNumber, runtime.Hasher, runtime.Header, runtime.Extrinsic]) { + // self.build_with_native_executor(None) + return tcb.BuildWithNativeExecutor(nil) +} + +// fn build_with_backend(self) -> (Client, Arc) { +// let backend = self.backend(); +// (self.build_with_native_executor(None).0, backend) +// } +// } diff --git a/internal/test-utils/runtime/runtime.go b/internal/test-utils/runtime/runtime.go new file mode 100644 index 0000000000..2faab6440a --- /dev/null +++ b/internal/test-utils/runtime/runtime.go @@ -0,0 +1,54 @@ +package runtime + +import ( + "github.com/ChainSafe/gossamer/internal/primitives/core/hash" + "github.com/ChainSafe/gossamer/internal/primitives/core/sr25519" + "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/runtime/generic" +) + +// / The address format for describing accounts. +// pub type Address = sp_core::sr25519::Public; +type Address = sr25519.Public + +// pub type Signature = sr25519::Signature; +type Signature = sr25519.Signature + +// / The extension to the basic transaction logic. +// pub type TxExtension = ( +// (CheckNonce, CheckWeight), +// CheckSubstrateCall, +// frame_metadata_hash_extension::CheckMetadataHash, +// ); + +type TxExtension struct{} + +// / Unchecked extrinsic type as expected by this runtime. +// pub type Extrinsic = +// +// sp_runtime::generic::UncheckedExtrinsic; +type Extrinsic = generic.UncheckedExtrinsic[Address, RuntimeCall, Signature, TxExtension] + +// / A simple hash type for all our hashing. +// pub type Hash = H256; +type Hash = hash.H256 + +// / The hashing algorithm used. +// pub type Hashing = BlakeTwo256; +type Hasher = runtime.BlakeTwo256 + +// / The block number type used in this runtime. +// pub type BlockNumber = u64; +type BlockNumber uint64 + +// / A test block. +// pub type Block = sp_runtime::generic::Block; +type Block = generic.Block[BlockNumber, Hash, Hasher, Extrinsic, Header] + +// / A test block's header. +// pub type Header = sp_runtime::generic::Header; +type Header = generic.Header[BlockNumber, Hash, Hasher] + +type RuntimeCall interface { + isRuntimeCall() +} diff --git a/internal/utils/fork-tree/fork_tree.go b/internal/utils/fork-tree/fork_tree.go new file mode 100644 index 0000000000..e268f9a3ba --- /dev/null +++ b/internal/utils/fork-tree/fork_tree.go @@ -0,0 +1,1071 @@ +package forktree + +import ( + "errors" + "iter" + "math" + "slices" + + "golang.org/x/exp/constraints" +) + +var ( + // Adding duplicate node to tree. + ErrDuplicate = errors.New("hash already exists in tree") + // Finalizing descendent of tree node without finalizing ancestor(s). + ErrUnfinalizedAncestor = errors.New("finalized descendent of tree node without finalizing its ancestor(s) first") + // Imported or finalized node that is an ancestor of previously finalized node. + ErrRevert = errors.New("tried to import or finalize node that is an ancestor of a previously finalized node") +) + +// Result of finalizing a node (that could be a part of the tree or not). +type FinalizationResult interface { + isFinalizationResult() +} + +// Filtering action. +type FilterAction uint + +const ( + // Remove the node and its subtree. + FilterActionRemove FilterAction = iota + 1 + // Maintain the node. + FilterActionKeepNode + // Maintain the node and its subtree. + FilterActionKeepTree +) + +// The tree has changed, optionally return the value associated with the finalized node. +type FinalizationResultChanged[V any] struct { + Value *V +} + +func (FinalizationResultChanged[V]) isFinalizationResult() {} + +// The tree has not changed. +type FinalizationResultUnchanged struct{} + +func (FinalizationResultUnchanged) isFinalizationResult() {} + +type node[H comparable, N constraints.Integer, V any] struct { + Hash H + Number N + Data V + Children []node[H, N, V] +} + +type nodeHeight[H comparable, N constraints.Integer, V any] struct { + node node[H, N, V] + height uint +} + +type nodeParentIdx[H comparable, N constraints.Integer, V any] struct { + node node[H, N, V] + parentIdx uint +} + +// Finds the max depth among all branches descendent from this node. +func (n node[H, N, V]) MaxDepth() uint { + var max uint = 0 + var stack []nodeHeight[H, N, V] = make([]nodeHeight[H, N, V], 0) + stack = append(stack, nodeHeight[H, N, V]{node: n, height: 0}) + + for len(stack) > 0 { + popped := stack[len(stack)-1] + stack = stack[:len(stack)-1] + node := popped.node + height := popped.height + if height > max { + max = height + } + + for _, child := range node.Children { + stack = append(stack, nodeHeight[H, N, V]{node: child, height: height + 1}) + } + } + return max +} + +// A tree data structure that stores several nodes across multiple branches. +// +// Top-level branches are called roots. The tree has functionality for +// finalizing nodes, which means that node is traversed, and all competing +// branches are pruned. It also guarantees that nodes in the tree are finalized +// in order. Each node is uniquely identified by its hash but can be ordered by +// its number. In order to build the tree an external function must be provided +// when interacting with the tree to establish a node's ancestry. +type ForkTree[H comparable, N constraints.Integer, V any] struct { + roots []node[H, N, V] + bestFinalizedNumber *N +} + +// Create a new empty tree instance. +func NewForkTree[H comparable, N constraints.Integer, V any]() ForkTree[H, N, V] { + return ForkTree[H, N, V]{ + roots: nil, + bestFinalizedNumber: nil, + } +} + +func (ft *ForkTree[H, N, V]) Clone() ForkTree[H, N, V] { + var bfn N + if ft.bestFinalizedNumber != nil { + bfn = *ft.bestFinalizedNumber + return ForkTree[H, N, V]{ + roots: slices.Clone(ft.roots), + bestFinalizedNumber: &bfn, + } + } + return ForkTree[H, N, V]{ + roots: slices.Clone(ft.roots), + bestFinalizedNumber: nil, + } +} + +// Rebalance the tree. +// +// For each tree level sort child nodes by max branch depth (decreasing). +// +// Most operations in the tree are performed with depth-first search +// starting from the leftmost node at every level, since this tree is meant +// to be used in a blockchain context, a good heuristic is that the node +// we'll be looking for at any point will likely be in one of the deepest chains +// (i.e. the longest ones). +func (ft *ForkTree[H, N, V]) Rebalance() { + slices.SortStableFunc(ft.roots, func(i, j node[H, N, V]) int { + iMaxDepth := i.MaxDepth() + jMaxDepth := j.MaxDepth() + if iMaxDepth > jMaxDepth { + return -1 + } else if iMaxDepth < jMaxDepth { + return 1 + } else { + return 0 + } + }) + stack := make([]*node[H, N, V], len(ft.roots)) + for i := range ft.roots { + stack[i] = &ft.roots[i] + } + + for len(stack) > 0 { + popped := stack[len(stack)-1] + stack = stack[:len(stack)-1] + slices.SortStableFunc(popped.Children, func(i, j node[H, N, V]) int { + iMaxDepth := i.MaxDepth() + jMaxDepth := j.MaxDepth() + if iMaxDepth > jMaxDepth { + return -1 + } else if iMaxDepth < jMaxDepth { + return 1 + } else { + return 0 + } + }) + for _, child := range popped.Children { + stack = append(stack, &child) + } + } +} + +// Import a new node into the tree. +// +// The given function `is_descendent_of` should return `true` if the second +// hash (target) is a descendent of the first hash (base). +// +// This method assumes that nodes in the same branch are imported in order. +// +// Returns `true` if the imported node is a root. +// WARNING: some users of this method (i.e. consensus epoch changes tree) currently silently +// rely on a **post-order DFS** traversal. If we are using instead a top-down traversal method +// then the `is_descendent_of` closure, when used after a warp-sync, may end up querying the +// backend for a block (the one corresponding to the root) that is not present and thus will +// return a wrong result. +func (ft *ForkTree[H, N, V]) Import( + hash H, + number N, + data V, + isDescendentOf func(a, b H) (bool, error), +) (bool, error) { + if ft.bestFinalizedNumber != nil { + if number <= *ft.bestFinalizedNumber { + return false, ErrRevert + } + } + + parent, err := ft.FindNodeWhereMut(hash, number, isDescendentOf, func(v V) bool { return true }) + if err != nil { + return false, err + } + var children *[]node[H, N, V] + var isRoot bool + if parent != nil { + children = &parent.Children + isRoot = false + } else { + children = &ft.roots + isRoot = true + } + + for _, elem := range *children { + if elem.Hash == hash { + return false, ErrDuplicate + } + } + + *children = append(*children, node[H, N, V]{ + Hash: hash, + Number: number, + Data: data, + Children: make([]node[H, N, V], 0), + }) + + if len(*children) == 1 { + // Rebalance may be required only if we've extended the branch depth. + ft.Rebalance() + } + + return isRoot, nil +} + +type Node[H comparable, N constraints.Integer, V any] struct { + Hash H + Number N + Data V +} + +// Iterates over the existing roots in the tree. +func (ft *ForkTree[H, N, V]) Roots() []Node[H, N, V] { + roots := make([]Node[H, N, V], len(ft.roots)) + for i, node := range ft.roots { + roots[i] = Node[H, N, V]{ + Hash: node.Hash, + Number: node.Number, + Data: node.Data, + } + } + return roots +} + +func (ft *ForkTree[H, N, V]) nodeIter() iter.Seq[node[H, N, V]] { + // we need to reverse the order of roots to maintain the expected + // ordering since the iterator uses a stack to track state. + stack := slices.Clone(ft.roots) + slices.Reverse(stack) + fti := forkTreeIterator[H, N, V]{ + stack: stack, + } + return fti.iter() +} + +func (ft *ForkTree[H, N, V]) Iter() iter.Seq[Node[H, N, V]] { + return func(yield func(Node[H, N, V]) bool) { + for node := range ft.nodeIter() { + if !yield(Node[H, N, V]{ + Hash: node.Hash, + Number: node.Number, + Data: node.Data, + }) { + return + } + } + } +} + +// Map fork tree into values of new types. +// +// Tree traversal technique (e.g. BFS vs DFS) is left as not specified and +// may be subject to change in the future. In other words, your predicates +// should not rely on the observed traversal technique currently in use. +func Map[H comparable, N constraints.Integer, V any, VT any]( + ft ForkTree[H, N, V], f func(H, N, V) VT, +) ForkTree[H, N, VT] { + var ( + queue []nodeParentIdx[H, N, V] = make([]nodeParentIdx[H, N, V], 0, len(ft.roots)) + nextQueue []nodeParentIdx[H, N, V] = make([]nodeParentIdx[H, N, V], 0) + output []nodeParentIdx[H, N, VT] = make([]nodeParentIdx[H, N, VT], 0) + ) + + for i := len(ft.roots) - 1; i >= 0; i-- { + queue = append(queue, nodeParentIdx[H, N, V]{ + node: ft.roots[i], + parentIdx: math.MaxUint, + }) + } + + for len(queue) > 0 { + for len(queue) > 0 { + popped := queue[0] + queue = queue[1:] + n := popped.node + parentIndex := popped.parentIdx + + newData := f(n.Hash, n.Number, n.Data) + newNode := node[H, N, VT]{ + Hash: n.Hash, + Number: n.Number, + Data: newData, + Children: make([]node[H, N, VT], 0, len(n.Children)), + } + + nodeID := uint(len(output)) + output = append(output, nodeParentIdx[H, N, VT]{node: newNode, parentIdx: parentIndex}) + + for i := len(n.Children) - 1; i >= 0; i-- { + child := n.Children[i] + nextQueue = append(nextQueue, nodeParentIdx[H, N, V]{node: child, parentIdx: nodeID}) + } + } + + temp := queue + queue = nextQueue + nextQueue = temp + } + + var roots []node[H, N, VT] + for len(output) > 0 { + elem := output[len(output)-1] + output = output[:len(output)-1] + parentIndex := elem.parentIdx + newNode := elem.node + if parentIndex == math.MaxUint { + roots = append(roots, newNode) + } else { + output[parentIndex].node.Children = append(output[parentIndex].node.Children, newNode) + } + } + + return ForkTree[H, N, VT]{ + roots: roots, + bestFinalizedNumber: ft.bestFinalizedNumber, + } +} + +// Find a node in the tree that is the deepest ancestor of the given +// block hash and which passes the given predicate. +// +// The given function `is_descendent_of` should return `true` if the +// second hash (target) is a descendent of the first hash (base). +func (ft *ForkTree[H, N, V]) FindNodeWhere( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), + predicate func(V) bool, +) (*node[H, N, V], error) { + maybePath, err := ft.FindNodeIndexWhere(hash, number, isDescendentOf, predicate) + if err != nil { + return nil, err + } + if maybePath == nil { + return nil, nil + } + var children []node[H, N, V] + curr := ft.roots + for _, i := range maybePath[:len(maybePath)-1] { + curr = curr[i].Children + } + children = curr + n := children[maybePath[len(maybePath)-1]] + return &n, nil +} + +// Same as [`find_node_where`](ForkTree::find_node_where), but returns mutable reference. +func (ft *ForkTree[H, N, V]) FindNodeWhereMut( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), + predicate func(V) bool, +) (*node[H, N, V], error) { + maybePath, err := ft.FindNodeIndexWhere(hash, number, isDescendentOf, predicate) + if err != nil { + return nil, err + } + if maybePath == nil { + return nil, nil + } + + var children *[]node[H, N, V] + curr := &ft.roots + for _, i := range maybePath[:len(maybePath)-1] { + curr = &(*curr)[i].Children + } + children = curr + return &(*children)[maybePath[len(maybePath)-1]], nil +} + +// Same as [`find_node_where`](ForkTree::find_node_where), but returns indices. +// +// The returned indices represent the full path to reach the matching node starting +// from one of the roots, i.e. the earliest index in the traverse path goes first, +// and the final index in the traverse path goes last. +// +// If a node is found that matches the predicate the returned path should always +// contain at least one index, otherwise `None` is returned. +// +// WARNING: some users of this method (i.e. consensus epoch changes tree) currently silently +// rely on a **post-order DFS** traversal. If we are using instead a top-down traversal method +// then the `is_descendent_of` closure, when used after a warp-sync, will end up querying the +// backend for a block (the one corresponding to the root) that is not present and thus will +// return a wrong result. +func (ft *ForkTree[H, N, V]) FindNodeIndexWhere( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), + predicate func(V) bool, +) ([]uint, error) { + var ( + stack []nodeParentIdx[H, N, V] = make([]nodeParentIdx[H, N, V], 0) + rootIdx uint = 0 + found bool = false + isDescendent bool = false + ) + + for rootIdx < uint(len(ft.roots)) { + if number <= ft.roots[rootIdx].Number { + rootIdx++ + continue + } + // The second element in the stack tuple tracks what is the **next** children + // index to search into. If we find an ancestor then we stop searching into + // alternative branches and we focus on the current path up to the root. + stack = append(stack, nodeParentIdx[H, N, V]{node: ft.roots[rootIdx], parentIdx: 0}) + for len(stack) > 0 { + popped := stack[len(stack)-1] + stack = stack[:len(stack)-1] + node := popped.node + i := popped.parentIdx + + if i < uint(len(node.Children)) && !isDescendent { + stack = append(stack, nodeParentIdx[H, N, V]{node: node, parentIdx: i + 1}) + if node.Children[i].Number < number { + stack = append(stack, nodeParentIdx[H, N, V]{node: node.Children[i], parentIdx: 0}) + } + } else { + if isDescendent { + isDescendent = true + if predicate(node.Data) { + found = true + break + } + } else { + is, err := isDescendentOf(node.Hash, hash) + if err != nil { + return nil, err + } + if is { + isDescendent = true + if predicate(node.Data) { + found = true + break + } + } + } + } + } + + // If the element we are looking for is a descendent of the current root + // then we can stop the search. + if isDescendent { + break + } + rootIdx++ + } + + if found { + // The path is the root index followed by the indices of all the children + // we were processing when we found the element (remember the stack + // contains the index of the **next** children to process). + path := make([]uint, 0, len(stack)+1) + path = append(path, rootIdx) + for _, elem := range stack { + path = append(path, elem.parentIdx-1) + } + return path, nil + } else { + return nil, nil + } +} + +// Prune the tree, removing all non-canonical nodes. +// +// We find the node in the tree that is the deepest ancestor of the given hash +// and that passes the given predicate. If such a node exists, we re-root the +// tree to this node. Otherwise the tree remains unchanged. +// +// The given function `is_descendent_of` should return `true` if the second +// hash (target) is a descendent of the first hash (base). +// +// Returns all pruned nodes data. +func (ft *ForkTree[H, N, V]) Prune( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), + predicate func(V) bool, +) (iter.Seq[Node[H, N, V]], error) { + path, err := ft.FindNodeIndexWhere(hash, number, isDescendentOf, predicate) + if err != nil { + return nil, err + } + if path == nil { + ri := removedIterator[H, N, V]{stack: nil} + return ri.iter(), nil + } + newRootPath := path + + removed := ft.roots + ft.roots = nil + + // Find and detach the new root from the removed nodes + rootSiblings := &removed + for _, idx := range newRootPath[:len(newRootPath)-1] { + rootSiblings = &(*rootSiblings)[idx].Children + } + root := (*rootSiblings)[newRootPath[len(newRootPath)-1]] + *rootSiblings = append((*rootSiblings)[:newRootPath[len(newRootPath)-1]], (*rootSiblings)[newRootPath[len(newRootPath)-1]+1:]...) + ft.roots = []node[H, N, V]{root} + + // If, because of the `predicate`, the new root is not the deepest ancestor + // of `hash` then we can remove all the nodes that are descendants of the new + // `root` but not ancestors of `hash`. + curr := &ft.roots[0] + for { + var maybeAncestorIdx *int + for idx, child := range curr.Children { + if child.Number < number { + is, err := isDescendentOf(child.Hash, hash) + if err != nil { + return nil, err + } + if is { + maybeAncestorIdx = &idx + break + } + } + } + if maybeAncestorIdx == nil { + // Now we are positioned just above block identified by `hash` + break + } + // Preserve only the ancestor node, the siblings are removed + nextSiblings := curr.Children + curr.Children = nil + next := nextSiblings[*maybeAncestorIdx] + nextSiblings = append(nextSiblings[:*maybeAncestorIdx], nextSiblings[*maybeAncestorIdx+1:]...) + curr.Children = []node[H, N, V]{next} + removed = append(removed, nextSiblings...) + curr = &curr.Children[0] + } + + // Curr now points to our direct ancestor, if necessary remove any node that is + // not a descendant of `hash`. + children := curr.Children + curr.Children = nil + for _, child := range children { + if child.Number == number && child.Hash == hash { + curr.Children = append(curr.Children, child) + } else if number < child.Number { + is, err := isDescendentOf(hash, child.Hash) + if err != nil { + return nil, err + } + if is { + curr.Children = append(curr.Children, child) + } else { + removed = append(removed, child) + } + } else { + removed = append(removed, child) + } + } + + ft.Rebalance() + + ri := removedIterator[H, N, V]{stack: removed} + return ri.iter(), nil +} + +// Finalize a root in the tree and return it, return `None` in case no root +// with the given hash exists. All other roots are pruned, and the children +// of the finalized node become the new roots. +func (ft *ForkTree[H, N, V]) FinalizeRoot(hash H) *V { + var pos int = -1 + for i, root := range ft.roots { + if root.Hash == hash { + pos = i + break + } + } + if pos < 0 { + return nil + } + v := ft.FinalizeRootAt(pos) + return &v +} + +// Finalize root at given position. See `finalize_root` comment for details. +func (ft *ForkTree[H, N, V]) FinalizeRootAt(position int) V { + node := ft.roots[position] + ft.roots = node.Children + ft.bestFinalizedNumber = &node.Number + return node.Data +} + +// Finalize a node in the tree. This method will make sure that the node +// being finalized is either an existing root (and return its data), or a +// node from a competing branch (not in the tree), tree pruning is done +// accordingly. The given function `is_descendent_of` should return `true` +// if the second hash (target) is a descendent of the first hash (base). +func (ft *ForkTree[H, N, V]) Finalize( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), +) (FinalizationResult, error) { + if ft.bestFinalizedNumber != nil { + if number <= *ft.bestFinalizedNumber { + return nil, ErrRevert + } + } + + root := ft.FinalizeRoot(hash) + if root != nil { + return FinalizationResultChanged[V]{root}, nil + } + + // make sure we're not finalizing a descendent of any root + for _, root := range ft.roots { + if number > root.Number { + is, err := isDescendentOf(root.Hash, hash) + if err != nil { + return nil, err + } + if is { + return nil, ErrUnfinalizedAncestor + } + } + } + + // we finalized a block earlier than any existing root (or possibly + // another fork not part of the tree). make sure to only keep roots that + // are part of the finalized branch + var changed bool + roots := ft.roots + ft.roots = nil + + for _, root := range roots { + if root.Number > number { + is, err := isDescendentOf(hash, root.Hash) + if err != nil { + return nil, err + } + if is { + ft.roots = append(ft.roots, root) + } else { + changed = true + } + } else { + changed = true + } + } + + ft.bestFinalizedNumber = &number + + if changed { + return FinalizationResultChanged[V]{nil}, nil + } + return FinalizationResultUnchanged{}, nil +} + +// Finalize a node in the tree and all its ancestors. The given function +// `is_descendent_of` should return `true` if the second hash (target) is +// a descendent of the first hash (base). +func (ft *ForkTree[H, N, V]) FinalizeWithAncestors( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), +) (FinalizationResult, error) { + if ft.bestFinalizedNumber != nil { + if number <= *ft.bestFinalizedNumber { + return nil, ErrRevert + } + } + + // check if one of the current roots is being finalized + root := ft.FinalizeRoot(hash) + if root != nil { + return FinalizationResultChanged[V]{root}, nil + } + + // we need to: + // 1) remove all roots that are not ancestors AND not descendants of finalized block; + // 2) if node is descendant - just leave it; + // 3) if node is ancestor - 'open it' + var ( + changed bool = false + idx int = 0 + ) + for idx != len(ft.roots) { + root := ft.roots[idx] + isFinalized := root.Hash == hash + var isDescendant bool = false + if !isFinalized && root.Number > number { + var err error + isDescendant, err = isDescendentOf(hash, root.Hash) + if err != nil { + return nil, err + } + } + var isAncestor bool = false + if !isFinalized && !isDescendant && root.Number < number { + var err error + isAncestor, err = isDescendentOf(root.Hash, hash) + if err != nil { + return nil, err + } + } + + // if we have met finalized root - open it and return + if isFinalized { + v := ft.FinalizeRootAt(idx) + return FinalizationResultChanged[V]{&v}, nil + } + + // if node is descendant of finalized block - just leave it as is + if isDescendant { + idx++ + continue + } + + // if node is ancestor of finalized block - remove it and continue with children + if isAncestor { + root := ft.roots[idx] + ft.roots = append(ft.roots[:idx], ft.roots[idx+1:]...) + ft.roots = append(ft.roots, root.Children...) + changed = true + continue + } + + // if node is neither ancestor, nor descendant of the finalized block - remove it + ft.roots = append(ft.roots[:idx], ft.roots[idx+1:]...) + changed = true + } + + ft.bestFinalizedNumber = &number + + if changed { + return FinalizationResultChanged[V]{nil}, nil + } + return FinalizationResultUnchanged{}, nil +} + +// Checks if any node in the tree is finalized by either finalizing the +// node itself or a node's descendent that's not in the tree, guaranteeing +// that the node being finalized isn't a descendent of (or equal to) any of +// the node's children. Returns `Some(true)` if the node being finalized is +// a root, `Some(false)` if the node being finalized is not a root, and +// `None` if no node in the tree is finalized. The given `predicate` is +// checked on the prospective finalized root and must pass for finalization +// to occur. The given function `is_descendent_of` should return `true` if +// the second hash (target) is a descendent of the first hash (base). +func (ft *ForkTree[H, N, V]) FinalizesAnyWithDescendentIf( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), + predicate func(V) bool, +) (*bool, error) { + if ft.bestFinalizedNumber != nil { + if number <= *ft.bestFinalizedNumber { + b := false + return &b, ErrRevert + } + } + + // check if the given hash is equal or a descendent of any node in the + // tree, if we find a valid node that passes the predicate then we must + // ensure that we're not finalizing past any of its child nodes. + for n := range ft.nodeIter() { + if predicate(n.Data) { + is, err := isDescendentOf(n.Hash, hash) + if err != nil { + var b bool = false + return &b, err + } + if n.Hash == hash || is { + for _, child := range n.Children { + if child.Number <= number { + if child.Hash == hash { + return nil, ErrUnfinalizedAncestor + } + is, err := isDescendentOf(child.Hash, hash) + if err != nil { + return nil, err + } + if is { + return nil, ErrUnfinalizedAncestor + } + } + } + + b := slices.ContainsFunc(ft.roots, func(root node[H, N, V]) bool { + return root.Hash == n.Hash + }) + return &b, nil + } + } + } + + return nil, nil +} + +// Finalize a root in the tree by either finalizing the node itself or a +// node's descendent that's not in the tree, guaranteeing that the node +// being finalized isn't a descendent of (or equal to) any of the root's +// children. The given `predicate` is checked on the prospective finalized +// root and must pass for finalization to occur. The given function +// `is_descendent_of` should return `true` if the second hash (target) is a +// descendent of the first hash (base). +func (ft *ForkTree[H, N, V]) FinalizeWithDescendentIf( + hash H, + number N, + isDescendentOf func(a, b H) (bool, error), + predicate func(V) bool, +) (FinalizationResult, error) { + if ft.bestFinalizedNumber != nil { + if number <= *ft.bestFinalizedNumber { + return nil, ErrRevert + } + } + + // check if the given hash is equal or a a descendent of any root, if we + // find a valid root that passes the predicate then we must ensure that + // we're not finalizing past any children node. + var position int = -1 + for i, root := range ft.roots { + if predicate(root.Data) { + is, err := isDescendentOf(root.Hash, hash) + if err != nil { + return nil, err + } + if root.Hash == hash || is { + for _, child := range root.Children { + if child.Number <= number { + if child.Hash == hash { + return nil, ErrUnfinalizedAncestor + } + is, err := isDescendentOf(child.Hash, hash) + if err != nil { + return nil, err + } + if is { + return nil, ErrUnfinalizedAncestor + } + } + } + + position = i + break + } + } + } + + var nodeData *V + if position >= 0 { + node := ft.roots[position] + ft.roots = node.Children + ft.bestFinalizedNumber = &node.Number + nodeData = &node.Data + } + + // Retain only roots that are descendants of the finalized block (this + // happens if the node has been properly finalized) or that are + // ancestors (or equal) to the finalized block (in this case the node + // wasn't finalized earlier presumably because the predicate didn't + // pass). + var changed bool = false + roots := ft.roots + ft.roots = nil + + for _, root := range roots { + var retain bool = false + if root.Number > number { + var err error + retain, err = isDescendentOf(hash, root.Hash) + if err != nil { + return nil, err + } + } else if root.Number == number && root.Hash == hash { + retain = true + } else { + var err error + retain, err = isDescendentOf(root.Hash, hash) + if err != nil { + return nil, err + } + } + + if retain { + ft.roots = append(ft.roots, root) + } else { + changed = true + } + } + + ft.bestFinalizedNumber = &number + + switch { + case nodeData != nil: + return FinalizationResultChanged[V]{nodeData}, nil + case changed: + return FinalizationResultChanged[V]{nil}, nil + case !changed: + return FinalizationResultUnchanged{}, nil + default: + panic("unreachable") + } +} + +// Remove from the tree some nodes (and their subtrees) using a `filter` predicate. +// +// The `filter` is called over tree nodes and returns a filter action: +// - `Remove` if the node and its subtree should be removed; +// - `KeepNode` if we should maintain the node and keep processing the tree. +// - `KeepTree` if we should maintain the node and its entire subtree. +// +// An iterator over all the pruned nodes is returned. +func (ft *ForkTree[H, N, V]) DrainFilter( + filter func(H, N, V) FilterAction, +) iter.Seq[Node[H, N, V]] { + // let mut removed = Vec::new(); + // let mut retained = Vec::new(); + removed := make([]node[H, N, V], 0) + retained := make([]nodeParentIdx[H, N, V], 0) + + roots := ft.roots + ft.roots = nil + queue := make([]nodeParentIdx[H, N, V], 0, len(roots)) + for i := len(roots) - 1; i >= 0; i-- { + queue = append(queue, nodeParentIdx[H, N, V]{parentIdx: math.MaxUint, node: roots[i]}) + } + var nextQueue []nodeParentIdx[H, N, V] + + for len(queue) > 0 { + for len(queue) > 0 { + popped := queue[0] + queue = queue[1:] + parentIdx := popped.parentIdx + n := popped.node + filterAction := filter(n.Hash, n.Number, n.Data) + switch filterAction { + case FilterActionKeepNode: + nodeIdx := len(retained) + children := n.Children + n.Children = nil + retained = append(retained, nodeParentIdx[H, N, V]{parentIdx: parentIdx, node: n}) + for i := len(children) - 1; i >= 0; i-- { + nextQueue = append(nextQueue, nodeParentIdx[H, N, V]{parentIdx: uint(nodeIdx), node: children[i]}) + } + case FilterActionKeepTree: + retained = append(retained, nodeParentIdx[H, N, V]{parentIdx: parentIdx, node: n}) + case FilterActionRemove: + removed = append(removed, n) + default: + panic("unreachable") + } + } + + temp := queue + queue = nextQueue + nextQueue = temp + } + + for len(retained) > 0 { + popped := retained[len(retained)-1] + retained = retained[:len(retained)-1] + parentIdx := popped.parentIdx + n := popped.node + if parentIdx == math.MaxUint { + ft.roots = append(ft.roots, n) + } else { + retained[parentIdx].node.Children = append(retained[parentIdx].node.Children, n) + } + } + + if len(removed) > 0 { + ft.Rebalance() + } + + ri := removedIterator[H, N, V]{stack: removed} + return ri.iter() +} + +type forkTreeIterator[H comparable, N constraints.Integer, V any] struct { + stack []node[H, N, V] +} + +func (fti *forkTreeIterator[H, N, V]) next() *node[H, N, V] { + if len(fti.stack) == 0 { + return nil + } + node := fti.stack[len(fti.stack)-1] + fti.stack = fti.stack[:len(fti.stack)-1] + // child nodes are stored ordered by max branch height (decreasing), + // we want to keep this ordering while iterating but since we're + // using a stack for iterator state we need to reverse it. + children := slices.Clone(node.Children) + slices.Reverse(children) + fti.stack = append(fti.stack, children...) + return &node +} + +func (fti *forkTreeIterator[H, N, V]) iter() iter.Seq[node[H, N, V]] { + return func(yield func(node[H, N, V]) bool) { + for { + item := fti.next() + if item == nil { + return + } + if !yield(*item) { + return + } + } + } +} + +type removedIterator[H comparable, N constraints.Integer, V any] struct { + // stack: Vec>, + stack []node[H, N, V] +} + +func (ri *removedIterator[H, N, V]) next() *Node[H, N, V] { + if len(ri.stack) == 0 { + return nil + } + node := ri.stack[len(ri.stack)-1] + ri.stack = ri.stack[:len(ri.stack)-1] + // child nodes are stored ordered by max branch height (decreasing), + // we want to keep this ordering while iterating but since we're + // using a stack for iterator state we need to reverse it. + children := node.Children + node.Children = nil + + slices.Reverse(children) + ri.stack = append(ri.stack, children...) + return &Node[H, N, V]{ + Hash: node.Hash, + Number: node.Number, + Data: node.Data, + } +} + +func (ri *removedIterator[H, N, V]) iter() iter.Seq[Node[H, N, V]] { + return func(yield func(Node[H, N, V]) bool) { + for { + item := ri.next() + if item == nil { + return + } + if !yield(*item) { + return + } + } + } +} diff --git a/internal/utils/fork-tree/fork_tree_test.go b/internal/utils/fork-tree/fork_tree_test.go new file mode 100644 index 0000000000..5aebea27dc --- /dev/null +++ b/internal/utils/fork-tree/fork_tree_test.go @@ -0,0 +1,957 @@ +package forktree + +import ( + "fmt" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func testForkTree(t *testing.T) (ForkTree[string, uint64, uint32], func(string, string) (bool, error)) { + t.Helper() + tree := NewForkTree[string, uint64, uint32]() + + // +---B-c-C---D---E + // | + // | +---G + // | | + // 0---A---F---H---I + // | | + // | +---L-m-M---N + // | | + // | +---O + // +---J---K + // + // (where N is not a part of fork tree) + // + // NOTE: the tree will get automatically rebalance on import and won't be laid out like the + // diagram above. the children will be ordered by subtree depth and the longest branches + // will be on the leftmost side of the tree. + var isDescendantOf = func(base, block string) (bool, error) { + letters := []string{"B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O"} + // This is a trick to have lowercase blocks be direct parents of their + // uppercase correspondent (A excluded) + block = strings.ToUpper(block) + switch { + case base == "A": + _, found := slices.BinarySearch(letters, block) + return found, nil + case base == "B" || base == "c": + return block == "C" || block == "D" || block == "E", nil + case base == "C": + return block == "D" || block == "E", nil + case base == "D": + return block == "E", nil + case base == "E": + return false, nil + case base == "F": + return block == "G" || block == "H" || block == "I" || block == "L" || block == "M" || block == "N" || block == "O", nil + case base == "G": + return false, nil + case base == "H": + return block == "I" || block == "L" || block == "M" || block == "N" || block == "O", nil + case base == "I": + return false, nil + case base == "J": + return block == "K", nil + case base == "K": + return false, nil + case base == "L": + return block == "M" || block == "N" || block == "O", nil + case base == "m": + return block == "M" || block == "N", nil + case base == "M": + return block == "N", nil + case base == "O": + return false, nil + case base == "0": + return true, nil + default: + return false, nil + } + } + + _, err := tree.Import("A", 10, 1, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("B", 20, 2, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("C", 30, 3, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("D", 40, 4, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("E", 50, 5, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("F", 20, 2, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("G", 30, 3, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("H", 30, 3, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("I", 40, 4, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("L", 40, 4, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("M", 50, 5, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("O", 50, 5, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("J", 20, 2, isDescendantOf) + require.NoError(t, err) + _, err = tree.Import("K", 30, 3, isDescendantOf) + require.NoError(t, err) + + return tree, isDescendantOf +} + +func TestForkTree(t *testing.T) { + t.Run("import_doesnt_revert", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + tree.FinalizeRoot("A") + + require.NotNil(t, tree.bestFinalizedNumber) + require.Equal(t, uint64(10), *tree.bestFinalizedNumber) + + _, err := tree.Import("A", 10, 1, isDescendentOf) + require.Error(t, err) + }) + + t.Run("import_doesnt_add_duplicates", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + _, err := tree.Import("A", 10, 1, isDescendentOf) + require.ErrorIs(t, err, ErrDuplicate) + + _, err = tree.Import("I", 40, 4, isDescendentOf) + require.ErrorIs(t, err, ErrDuplicate) + + _, err = tree.Import("G", 30, 3, isDescendentOf) + require.ErrorIs(t, err, ErrDuplicate) + + _, err = tree.Import("K", 30, 3, isDescendentOf) + require.ErrorIs(t, err, ErrDuplicate) + }) + + type hn struct { + H string + N uint64 + } + t.Run("finalize_root_works", func(t *testing.T) { + var finalizeA = func() ForkTree[string, uint64, uint32] { + tree, _ := testForkTree(t) + + var actual []hn + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "A", N: 10}}, actual) + + require.Nil(t, tree.bestFinalizedNumber) + + // finalizing "A" opens up three possible forks + tree.FinalizeRoot("A") + + actual = nil + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "B", N: 20}, {H: "F", N: 20}, {H: "J", N: 20}}, actual) + + return tree + } + + { + tree := finalizeA() + + // finalizing "B" will progress on its fork and remove any other competing forks + tree.FinalizeRoot("B") + + var actual []hn + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "C", N: 30}}, actual) + // all the other forks have been pruned + require.Len(t, tree.roots, 1) + } + + { + tree := finalizeA() + + // finalizing "J" will progress on its fork and remove any other competing forks + tree.FinalizeRoot("J") + + var actual []hn + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "K", N: 30}}, actual) + // all the other forks have been pruned + require.Len(t, tree.roots, 1) + } + }) + + t.Run("finalize_works", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + originalRoots := slices.Clone(tree.roots) + + // finalizing a block prior to any in the node doesn't change the tree + res, err := tree.Finalize("0", 0, isDescendentOf) + require.NoError(t, err) + require.Equal(t, FinalizationResultUnchanged{}, res) + + require.Equal(t, originalRoots, tree.roots) + + // finalizing "A" opens up three possible forks + res, err = tree.Finalize("A", 10, isDescendentOf) + require.NoError(t, err) + v := uint32(1) + require.Equal(t, FinalizationResultChanged[uint32]{Value: &v}, res) + + var actual []hn + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "B", N: 20}, {H: "F", N: 20}, {H: "J", N: 20}}, actual) + + // finalizing anything lower than what we observed will fail + require.NotNil(t, tree.bestFinalizedNumber) + require.Equal(t, uint64(10), *tree.bestFinalizedNumber) + + _, err = tree.Finalize("Z", 10, isDescendentOf) + require.ErrorIs(t, err, ErrRevert) + + // trying to finalize a node without finalizing its ancestors first will fail + _, err = tree.Finalize("H", 30, isDescendentOf) + require.ErrorIs(t, err, ErrUnfinalizedAncestor) + + // after finalizing "F" we can finalize "H" + res, err = tree.Finalize("F", 20, isDescendentOf) + require.NoError(t, err) + v = uint32(2) + require.Equal(t, FinalizationResultChanged[uint32]{Value: &v}, res) + + res, err = tree.Finalize("H", 30, isDescendentOf) + require.NoError(t, err) + v = uint32(3) + require.Equal(t, FinalizationResultChanged[uint32]{Value: &v}, res) + + actual = nil + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "L", N: 40}, {H: "I", N: 40}}, actual) + + // finalizing a node from another fork that isn't part of the tree clears the tree + res, err = tree.Finalize("Z", 50, isDescendentOf) + require.NoError(t, err) + require.Equal(t, FinalizationResultChanged[uint32]{Value: nil}, res) + + require.Empty(t, tree.roots) + }) + + t.Run("finalize_with_ancestor_works", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + originalRoots := slices.Clone(tree.roots) + + // finalizing a block prior to any in the node doesn't change the tree + res, err := tree.FinalizeWithAncestors("0", 0, isDescendentOf) + require.NoError(t, err) + require.Equal(t, FinalizationResultUnchanged{}, res) + + require.Equal(t, originalRoots, tree.roots) + + // finalizing "A" opens up three possible forks + res, err = tree.FinalizeWithAncestors("A", 10, isDescendentOf) + require.NoError(t, err) + v := uint32(1) + require.Equal(t, FinalizationResultChanged[uint32]{Value: &v}, res) + + var actual []hn + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "B", N: 20}, {H: "F", N: 20}, {H: "J", N: 20}}, actual) + + // finalizing H: + // 1) removes roots that are not ancestors/descendants of H (B, J) + // 2) opens root that is ancestor of H (F -> G+H) + // 3) finalizes the just opened root H (H -> I + L) + res, err = tree.FinalizeWithAncestors("H", 30, isDescendentOf) + require.NoError(t, err) + v = uint32(3) + require.Equal(t, FinalizationResultChanged[uint32]{Value: &v}, res) + + actual = nil + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "L", N: 40}, {H: "I", N: 40}}, actual) + + require.NotNil(t, tree.bestFinalizedNumber) + require.Equal(t, uint64(30), *tree.bestFinalizedNumber) + + // finalizing N (which is not a part of the tree): + // 1) removes roots that are not ancestors/descendants of N (I) + // 2) opens root that is ancestor of N (L -> M+O) + // 3) removes roots that are not ancestors/descendants of N (O) + // 4) opens root that is ancestor of N (M -> {}) + res, err = tree.FinalizeWithAncestors("N", 60, isDescendentOf) + require.NoError(t, err) + require.Equal(t, FinalizationResultChanged[uint32]{Value: nil}, res) + + actual = nil + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Empty(t, actual) + + require.NotNil(t, tree.bestFinalizedNumber) + require.Equal(t, uint64(60), *tree.bestFinalizedNumber) + }) + + t.Run("finalize_with_descendent_works", func(t *testing.T) { + type Change struct { + Effective uint64 + } + tree := NewForkTree[string, uint64, Change]() + isDescendentOf := func(base, block string) (bool, error) { + // A0 #1 - (B #2) - (C #5) - D #10 - E #15 - (F #100) + // \ + // - (G #100) + // + // A1 #1 + // + // Nodes B, C, F and G are not part of the tree. + switch { + case base == "A0": + return block == "B" || block == "C" || block == "D" || block == "E" || block == "G", nil + case base == "A1": + return false, nil + case base == "C": + return block == "D", nil + case base == "D": + return block == "E" || block == "F" || block == "G", nil + case base == "E": + return block == "F", nil + default: + return false, nil + } + } + + isRoot, err := tree.Import("A0", 1, Change{Effective: 5}, isDescendentOf) + require.NoError(t, err) + require.True(t, isRoot) + isRoot, err = tree.Import("A1", 1, Change{Effective: 5}, isDescendentOf) + require.NoError(t, err) + require.True(t, isRoot) + isRoot, err = tree.Import("D", 10, Change{Effective: 10}, isDescendentOf) + require.NoError(t, err) + require.False(t, isRoot) + isRoot, err = tree.Import("E", 15, Change{Effective: 50}, isDescendentOf) + require.NoError(t, err) + require.False(t, isRoot) + + res, err := tree.FinalizesAnyWithDescendentIf("B", 2, isDescendentOf, func(c Change) bool { + return c.Effective <= 2 + }) + require.NoError(t, err) + require.Nil(t, res) + + // finalizing "D" is not allowed since it is not a root. + _, err = tree.FinalizeWithDescendentIf("D", 10, isDescendentOf, func(c Change) bool { + return c.Effective <= 10 + }) + require.ErrorIs(t, err, ErrUnfinalizedAncestor) + + // finalizing "D" will finalize a block from the tree, but it can't be applied yet + // since it is not a root change. + res, err = tree.FinalizesAnyWithDescendentIf("D", 10, isDescendentOf, func(c Change) bool { + return c.Effective == 10 + }) + require.NoError(t, err) + require.NotNil(t, res) + require.False(t, *res) + + // finalizing "E" is not allowed since there are not finalized ancestors. + _, err = tree.FinalizesAnyWithDescendentIf("E", 15, isDescendentOf, func(c Change) bool { + return c.Effective == 10 + }) + require.ErrorIs(t, err, ErrUnfinalizedAncestor) + + // finalizing "B" doesn't finalize "A0" since the predicate doesn't pass, + // although it will clear out "A1" from the tree + result, err := tree.FinalizeWithDescendentIf("B", 2, isDescendentOf, func(c Change) bool { + return c.Effective <= 2 + }) + require.NoError(t, err) + require.Equal(t, FinalizationResultChanged[Change]{Value: nil}, result) + + var actual []hn + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "A0", N: 1}}, actual) + + // finalizing "C" will finalize the node "A0" and prune it out of the tree + res, err = tree.FinalizesAnyWithDescendentIf("C", 5, isDescendentOf, func(c Change) bool { + return c.Effective <= 5 + }) + require.NoError(t, err) + require.NotNil(t, res) + require.True(t, *res) + + result, err = tree.FinalizeWithDescendentIf("C", 5, isDescendentOf, func(c Change) bool { + return c.Effective <= 5 + }) + require.NoError(t, err) + require.Equal(t, FinalizationResultChanged[Change]{Value: &Change{Effective: 5}}, result) + + actual = nil + for _, node := range tree.Roots() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + require.Equal(t, []hn{{H: "D", N: 10}}, actual) + + // finalizing "F" will fail since it would finalize past "E" without finalizing "D" first + _, err = tree.FinalizesAnyWithDescendentIf("F", 100, isDescendentOf, func(c Change) bool { + return c.Effective <= 100 + }) + require.ErrorIs(t, err, ErrUnfinalizedAncestor) + + // it will work with "G" though since it is not in the same branch as "E" + res, err = tree.FinalizesAnyWithDescendentIf("G", 100, isDescendentOf, func(c Change) bool { + return c.Effective <= 100 + }) + require.NoError(t, err) + require.NotNil(t, res) + require.True(t, *res) + + result, err = tree.FinalizeWithDescendentIf("G", 100, isDescendentOf, func(c Change) bool { + return c.Effective <= 100 + }) + require.NoError(t, err) + require.Equal(t, FinalizationResultChanged[Change]{Value: &Change{Effective: 10}}, result) + + // "E" will be pruned out + require.Empty(t, tree.roots) + }) + + t.Run("iter_iterates_in_preorder", func(t *testing.T) { + tree, _ := testForkTree(t) + + actual := make([]hn, 0) + for node := range tree.Iter() { + actual = append(actual, hn{H: node.Hash, N: node.Number}) + } + expected := []hn{ + {H: "A", N: 10}, + {H: "B", N: 20}, + {H: "C", N: 30}, + {H: "D", N: 40}, + {H: "E", N: 50}, + {H: "F", N: 20}, + {H: "H", N: 30}, + {H: "L", N: 40}, + {H: "M", N: 50}, + {H: "O", N: 50}, + {H: "I", N: 40}, + {H: "G", N: 30}, + {H: "J", N: 20}, + {H: "K", N: 30}, + } + require.Equal(t, expected, actual) + }) + + t.Run("minimizes_calls_to_is_descendent_of", func(t *testing.T) { + var nIsDescendentOfCalls uint + + isDescendentOf := func(base, block string) (bool, error) { + nIsDescendentOfCalls++ + return true, nil + } + + { + // Deep tree where we want to call `finalizes_any_with_descendent_if`. The + // search for the node should first check the predicate (which is cheaper) and + // only then call `is_descendent_of` + tree := NewForkTree[string, uint, uint]() + letters := []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K"} + + for i, letter := range letters { + _, err := tree.Import(letter, uint(i), uint(i), func(base, block string) (bool, error) { + return true, nil + }) + require.NoError(t, err) + } + + // "L" is a descendent of "K", but the predicate will only pass for "K", + // therefore only one call to `is_descendent_of` should be made + res, err := tree.FinalizesAnyWithDescendentIf("L", 11, isDescendentOf, func(i uint) bool { + return i == 10 + }) + require.NoError(t, err) + require.NotNil(t, res) + require.False(t, *res) + + require.Equal(t, uint(1), nIsDescendentOfCalls) + } + + nIsDescendentOfCalls = 0 + + { + // Multiple roots in the tree where we want to call `finalize_with_descendent_if`. + // The search for the root node should first check the predicate (which is cheaper) + // and only then call `is_descendent_of` + tree := NewForkTree[string, uint, uint]() + letters := []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K"} + + for i, letter := range letters { + _, err := tree.Import(letter, uint(i), uint(i), func(base, block string) (bool, error) { + return false, nil + }) + require.NoError(t, err) + } + + // "L" is a descendent of "K", but the predicate will only pass for "K", + // therefore only one call to `is_descendent_of` should be made + res, err := tree.FinalizeWithDescendentIf("L", 11, isDescendentOf, func(i uint) bool { + return i == 10 + }) + require.NoError(t, err) + v := uint(10) + require.Equal(t, FinalizationResultChanged[uint]{Value: &v}, res) + + require.Equal(t, uint(1), nIsDescendentOfCalls) + } + }) + + t.Run("map_works", func(t *testing.T) { + tree, _ := testForkTree(t) + + // Extend the single root fork-tree to also exercise the roots order during map. + isDescendentOf := func(base, block string) (bool, error) { + return false, nil + } + isRoot, err := tree.Import("A1", 10, 1, isDescendentOf) + require.NoError(t, err) + require.True(t, isRoot) + isRoot, err = tree.Import("A2", 10, 1, isDescendentOf) + require.NoError(t, err) + require.True(t, isRoot) + + oldTree := tree.Clone() + newTree := Map[string, uint64, uint32, string](tree, func(hash string, number uint64, data uint32) string { + return hash + }) + require.Equal(t, len(oldTree.Roots()), len(newTree.Roots())) + + // Check content and order + for node := range newTree.Iter() { + require.Equal(t, node.Hash, node.Data) + } + oldTreeHashes := make([]string, 0) + for node := range oldTree.Iter() { + oldTreeHashes = append(oldTreeHashes, node.Hash) + } + newTreeHashes := make([]string, 0) + for node := range newTree.Iter() { + newTreeHashes = append(newTreeHashes, node.Hash) + } + require.Equal(t, oldTreeHashes, newTreeHashes) + }) + + t.Run("prune_works_for_in_tree_hashes", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + removed, err := tree.Prune("C", 30, isDescendentOf, func(height uint32) bool { + return true + }) + require.NoError(t, err) + actual := make([]string, 0) + for _, node := range tree.Roots() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"B"}, actual) + + actual = nil + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"B", "C", "D", "E"}, actual) + + removedHashes := make([]string, 0) + for node := range removed { + removedHashes = append(removedHashes, node.Hash) + } + require.Equal(t, []string{"A", "F", "H", "L", "M", "O", "I", "G", "J", "K"}, removedHashes) + + removed, err = tree.Prune("E", 50, isDescendentOf, func(height uint32) bool { + return true + }) + + actual = nil + for _, node := range tree.Roots() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"D"}, actual) + + actual = nil + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"D", "E"}, actual) + + removedHashes = nil + for node := range removed { + removedHashes = append(removedHashes, node.Hash) + } + require.Equal(t, []string{"B", "C"}, removedHashes) + }) + + t.Run("prune_works_for_out_of_tree_hashes", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + removed, err := tree.Prune("c", 25, isDescendentOf, func(height uint32) bool { + return true + }) + require.NoError(t, err) + + actual := make([]string, 0) + for _, node := range tree.Roots() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"B"}, actual) + + actual = nil + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"B", "C", "D", "E"}, actual) + + removedHashes := make([]string, 0) + for node := range removed { + removedHashes = append(removedHashes, node.Hash) + } + require.Equal(t, []string{"A", "F", "H", "L", "M", "O", "I", "G", "J", "K"}, removedHashes) + }) + + t.Run("prune_works_for_not_direct_ancestor", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + // This is to re-root the tree not at the immediate ancestor, but the one just before. + removed, err := tree.Prune("m", 45, isDescendentOf, func(height uint32) bool { + return height == 3 + }) + require.NoError(t, err) + + actual := make([]string, 0) + for _, node := range tree.Roots() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"H"}, actual) + + actual = nil + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"H", "L", "M"}, actual) + + removedHashes := make([]string, 0) + for node := range removed { + removedHashes = append(removedHashes, node.Hash) + } + require.Equal(t, []string{"O", "I", "A", "B", "C", "D", "E", "F", "G", "J", "K"}, removedHashes) + }) + + t.Run("prune_works_for_far_away_ancestor", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + removed, err := tree.Prune("m", 45, isDescendentOf, func(height uint32) bool { + return height == 2 + }) + require.NoError(t, err) + + actual := make([]string, 0) + for _, node := range tree.Roots() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"F"}, actual) + + actual = nil + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"F", "H", "L", "M"}, actual) + + removedHashes := make([]string, 0) + for node := range removed { + removedHashes = append(removedHashes, node.Hash) + } + require.Equal(t, []string{"O", "I", "G", "A", "B", "C", "D", "E", "J", "K"}, removedHashes) + }) + + t.Run("find_node_backtracks_after_finding_highest_descending_node", func(t *testing.T) { + tree := NewForkTree[string, int32, int32]() + + // A - B + // \ + // — C + isDescendentOf := func(base, block string) (bool, error) { + switch base { + case "A": + return block == "B" || block == "C" || block == "D", nil + case "B", "C": + return block == "D", nil + case "0": + return true, nil + default: + return false, nil + } + } + + _, err := tree.Import("A", 1, 1, isDescendentOf) + require.NoError(t, err) + _, err = tree.Import("B", 2, 2, isDescendentOf) + require.NoError(t, err) + _, err = tree.Import("C", 2, 4, isDescendentOf) + require.NoError(t, err) + + // when searching the tree we reach node `C`, but the + // predicate doesn't pass. we should backtrack to `B`, but not to `A`, + // since "B" fulfills the predicate. + node, err := tree.FindNodeWhere("D", 3, isDescendentOf, func(data int32) bool { + return data < 3 + }) + require.NoError(t, err) + require.NotNil(t, node) + + require.Equal(t, "B", node.Hash) + }) + + t.Run("rebalance_works", func(t *testing.T) { + tree, _ := testForkTree(t) + + // the tree is automatically rebalanced on import, therefore we should iterate in preorder + // exploring the longest forks first. check the ascii art above to understand the expected + // output below. + actual := make([]string, 0) + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"A", "B", "C", "D", "E", "F", "H", "L", "M", "O", "I", "G", "J", "K"}, actual) + + // let's add a block "P" which is a descendent of block "O" + isDescendentOf := func(base, block string) (bool, error) { + if block == "P" { + return base == "A" || base == "F" || base == "H" || base == "L" || base == "O", nil + } + return false, nil + } + + _, err := tree.Import("P", 60, 6, isDescendentOf) + require.NoError(t, err) + + // this should re-order the tree, since the branch "A -> B -> C -> D -> E" is no longer tied + // with 5 blocks depth. additionally "O" should be visited before "M" now, since it has one + // descendent "P" which makes that branch 6 blocks long. + actual = nil + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"A", "F", "H", "L", "O", "P", "M", "I", "G", "B", "C", "D", "E", "J", "K"}, actual) + }) + + t.Run("drain_filter_works", func(t *testing.T) { + tree, _ := testForkTree(t) + + filter := func(h string, n uint64, d uint32) FilterAction { + switch h { + case "A", "B", "F", "G": + return FilterActionKeepNode + case "C": + return FilterActionKeepTree + case "H", "J": + return FilterActionRemove + default: + panic(fmt.Sprintf("Unexpected filtering for node: %s", h)) + } + } + + removed := tree.DrainFilter(filter) + + actual := make([]string, 0) + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"A", "B", "C", "D", "E", "F", "G"}, actual) + + removedHashes := make([]string, 0) + for node := range removed { + removedHashes = append(removedHashes, node.Hash) + } + require.Equal(t, []string{"H", "L", "M", "O", "I", "J", "K"}, removedHashes) + }) + + t.Run("find_node_index_works", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + path, err := tree.FindNodeIndexWhere("D", 40, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 0, 0}, path) + + path, err = tree.FindNodeIndexWhere("O", 50, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 1, 0, 0}, path) + + path, err = tree.FindNodeIndexWhere("N", 60, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 1, 0, 0, 0}, path) + }) + + t.Run("find_node_index_with_predicate_works", func(t *testing.T) { + isDescendentOf := func(parent, child rune) (bool, error) { + switch parent { + case 'A': + return child == 'B' || child == 'C' || child == 'D' || child == 'E' || child == 'F', nil + case 'B': + return child == 'C' || child == 'D', nil + case 'C': + return child == 'D', nil + case 'E': + return child == 'F', nil + case 'D', 'F': + return false, nil + default: + return false, fmt.Errorf("TestError") + } + } + + // A(t) --- B(f) --- C(t) --- D(f) + // \-- E(t) --- F(f) + tree := NewForkTree[rune, uint8, bool]() + _, err := tree.Import('A', 1, true, isDescendentOf) + require.NoError(t, err) + _, err = tree.Import('B', 2, false, isDescendentOf) + require.NoError(t, err) + _, err = tree.Import('C', 3, true, isDescendentOf) + require.NoError(t, err) + _, err = tree.Import('D', 4, false, isDescendentOf) + require.NoError(t, err) + + _, err = tree.Import('E', 2, true, isDescendentOf) + require.NoError(t, err) + _, err = tree.Import('F', 3, false, isDescendentOf) + require.NoError(t, err) + + path, err := tree.FindNodeIndexWhere('D', 4, isDescendentOf, func(value bool) bool { + return !value + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 0}, path) + + path, err = tree.FindNodeIndexWhere('D', 4, isDescendentOf, func(value bool) bool { + return value + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 0, 0}, path) + + path, err = tree.FindNodeIndexWhere('F', 3, isDescendentOf, func(value bool) bool { + return !value + }) + require.NoError(t, err) + require.Nil(t, path) + + path, err = tree.FindNodeIndexWhere('F', 3, isDescendentOf, func(value bool) bool { + return value + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 1}, path) + }) + + t.Run("find_node_works", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + node, err := tree.FindNodeWhere("B", 20, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, node) + require.Equal(t, "A", node.Hash) + require.Equal(t, uint64(10), node.Number) + + node, err = tree.FindNodeWhere("D", 40, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, node) + require.Equal(t, "C", node.Hash) + require.Equal(t, uint64(30), node.Number) + + node, err = tree.FindNodeWhere("O", 50, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, node) + require.Equal(t, "L", node.Hash) + require.Equal(t, uint64(40), node.Number) + + node, err = tree.FindNodeWhere("N", 60, isDescendentOf, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, node) + require.Equal(t, "M", node.Hash) + require.Equal(t, uint64(50), node.Number) + }) + + t.Run("post_order_traversal_requirement", func(t *testing.T) { + tree, isDescendentOf := testForkTree(t) + + // Test for the post-order DFS traversal requirement as specified by the + // `find_node_index_where` and `import` comments. + isDescendentOfForPostOrder := func(parent, child string) (bool, error) { + if parent == "A" { + return false, fmt.Errorf("TestError") + } + if parent == "K" && child == "Z" { + return true, nil + } + return isDescendentOf(parent, child) + } + + // Post order traversal requirement for `find_node_index_where` + path, err := tree.FindNodeIndexWhere("N", 60, isDescendentOfForPostOrder, func(data uint32) bool { + return true + }) + require.NoError(t, err) + require.NotNil(t, path) + require.Equal(t, []uint{0, 1, 0, 0, 0}, path) + + // Post order traversal requirement for `import` + res, err := tree.Import("Z", 100, 10, isDescendentOfForPostOrder) + require.NoError(t, err) + require.False(t, res) + actual := make([]string, 0) + for node := range tree.Iter() { + actual = append(actual, node.Hash) + } + require.Equal(t, []string{"A", "B", "C", "D", "E", "F", "H", "L", "M", "O", "I", "G", "J", "K", "Z"}, actual) + }) +} diff --git a/lib/grandpa/message_handler.go b/lib/grandpa/message_handler.go index 233b63fd18..d19a786834 100644 --- a/lib/grandpa/message_handler.go +++ b/lib/grandpa/message_handler.go @@ -11,6 +11,7 @@ import ( "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/internal/database" + "github.com/ChainSafe/gossamer/internal/primitives/consensus/grandpa" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" "github.com/ChainSafe/gossamer/internal/primitives/runtime" "github.com/ChainSafe/gossamer/internal/primitives/runtime/generic" @@ -380,10 +381,10 @@ func (s *Service) VerifyBlockJustification(finalizedHash common.Hash, finalizedN logger.Debugf("verifying justification within set id %d and authorities %d", setID, len(auths)) - idsAndWeights := make([]finality_grandpa.IDWeight[string], len(auths)) + idsAndWeights := make([]finality_grandpa.IDWeight[grandpa.AuthorityID], len(auths)) for idx, auth := range auths { - idsAndWeights[idx] = finality_grandpa.IDWeight[string]{ - ID: string(auth.Key.Encode()), + idsAndWeights[idx] = finality_grandpa.IDWeight[grandpa.AuthorityID]{ + ID: grandpa.AuthorityID(auth.Key.Encode()), Weight: 1, } } diff --git a/lib/grandpa/warpsync/warp_sync_test.go b/lib/grandpa/warpsync/warp_sync_test.go index 6fd70aa5bf..1adb56f69b 100644 --- a/lib/grandpa/warpsync/warp_sync_test.go +++ b/lib/grandpa/warpsync/warp_sync_test.go @@ -415,22 +415,22 @@ func mapDigest(t *testing.T, digest types.Digest) runtime.Digest { switch v := value.(type) { case types.PreRuntimeDigest: - newDigest.Push(runtime.NewDigestItem(runtime.PreRuntime{ + newDigest.Push(runtime.DigestItemPreRuntime{ ConsensusEngineID: runtime.ConsensusEngineID(v.ConsensusEngineID), Bytes: v.Data, - })) + }) case types.ConsensusDigest: - newDigest.Push(runtime.NewDigestItem(runtime.Consensus{ + runtime.NewDigestItemVDT(runtime.DigestItemConsensus{ ConsensusEngineID: runtime.ConsensusEngineID(v.ConsensusEngineID), Bytes: v.Data, - })) + }) case types.SealDigest: - newDigest.Push(runtime.NewDigestItem(runtime.Seal{ + runtime.NewDigestItemVDT(runtime.DigestItemSeal{ ConsensusEngineID: runtime.ConsensusEngineID(v.ConsensusEngineID), Bytes: v.Data, - })) + }) case types.RuntimeEnvironmentUpdated: - newDigest.Push(runtime.NewDigestItem(runtime.RuntimeEnvironmentUpdated{})) + runtime.NewDigestItemVDT(runtime.DigestItemRuntimeEnvironmentUpdated{}) } } diff --git a/pkg/finality-grandpa/environment_test.go b/pkg/finality-grandpa/environment_test.go index 22decad5b1..729a8ae987 100644 --- a/pkg/finality-grandpa/environment_test.go +++ b/pkg/finality-grandpa/environment_test.go @@ -266,11 +266,11 @@ func (rn *RoundNetwork) AddNode( } type GlobalMessageNetwork struct { - *BroadcastNetwork[globalInItem[string, uint32, Signature, ID], CommunicationOut[string, uint32, Signature, ID]] + *BroadcastNetwork[GlobalInItem[string, uint32, Signature, ID], CommunicationOut[string, uint32, Signature, ID]] } func NewGlobalMessageNetwork() *GlobalMessageNetwork { - bn := NewBroadcastNetwork[globalInItem[ + bn := NewBroadcastNetwork[GlobalInItem[ string, uint32, Signature, ID], CommunicationOut[string, uint32, Signature, ID], ]() gmn := GlobalMessageNetwork{bn} @@ -278,9 +278,9 @@ func NewGlobalMessageNetwork() *GlobalMessageNetwork { } func (gmn *GlobalMessageNetwork) AddNode( - f func(CommunicationOut[string, uint32, Signature, ID]) globalInItem[string, uint32, Signature, ID], + f func(CommunicationOut[string, uint32, Signature, ID]) GlobalInItem[string, uint32, Signature, ID], out chan CommunicationOut[string, uint32, Signature, ID], -) (in chan globalInItem[string, uint32, Signature, ID]) { +) (in chan GlobalInItem[string, uint32, Signature, ID]) { return gmn.BroadcastNetwork.AddNode(f, out) } @@ -332,12 +332,12 @@ func (n *Network) MakeRoundComms( func (n *Network) MakeGlobalComms( out chan CommunicationOut[string, uint32, Signature, ID], -) chan globalInItem[string, uint32, Signature, ID] { +) chan GlobalInItem[string, uint32, Signature, ID] { n.mtx.Lock() defer n.mtx.Unlock() return n.globalMessages.AddNode( - func(message CommunicationOut[string, uint32, Signature, ID]) globalInItem[string, uint32, Signature, ID] { + func(message CommunicationOut[string, uint32, Signature, ID]) GlobalInItem[string, uint32, Signature, ID] { if message == nil { panic("nil message variant") } @@ -348,7 +348,7 @@ func (n *Network) MakeGlobalComms( CompactCommit: message.Commit.CompactCommit(), Callback: nil, } - return globalInItem[string, uint32, Signature, ID]{ + return GlobalInItem[string, uint32, Signature, ID]{ CommunicationIn: ci, } default: @@ -358,5 +358,5 @@ func (n *Network) MakeGlobalComms( } func (n *Network) SendMessage(message CommunicationIn[string, uint32, Signature, ID]) { - n.globalMessages.SendMessage(globalInItem[string, uint32, Signature, ID]{message, nil}) + n.globalMessages.SendMessage(GlobalInItem[string, uint32, Signature, ID]{message, nil}) } diff --git a/pkg/finality-grandpa/voter.go b/pkg/finality-grandpa/voter.go index 6d9e751607..50d3ac6082 100644 --- a/pkg/finality-grandpa/voter.go +++ b/pkg/finality-grandpa/voter.go @@ -362,17 +362,6 @@ type CommunicationOut[ isCommunicationOut() } -// CommuincationOutVariants is interface constraint of `CommunicationOut` -type CommuincationOutVariants[ - Hash constraints.Ordered, - Number constraints.Unsigned, - Signature comparable, - ID constraints.Ordered, -] interface { - CommunicationOutCommit[Hash, Number, Signature, ID] - CommunicationOut[Hash, Number, Signature, ID] -} - // CommunicationOutCommit is a commit message. type CommunicationOutCommit[ Hash constraints.Ordered, @@ -480,16 +469,6 @@ type CommunicationIn[ ] interface { isCommunicationIn() } - -type CommunicationInVariants[ - Hash constraints.Ordered, - Number constraints.Unsigned, - Signature comparable, - ID constraints.Ordered, -] interface { - CommunicationInCommit[Hash, Number, Signature, ID] | CommunicationInCatchUp[Hash, Number, Signature, ID] - CommunicationIn[Hash, Number, Signature, ID] -} type CommunicationInCommit[ Hash constraints.Ordered, Number constraints.Unsigned, @@ -515,7 +494,7 @@ type CommunicationInCatchUp[ func (CommunicationInCatchUp[Hash, Number, Signature, ID]) isCommunicationIn() {} -type globalInItem[ +type GlobalInItem[ Hash constraints.Ordered, Number constraints.Unsigned, Signature comparable, @@ -549,7 +528,7 @@ type Voter[Hash constraints.Ordered, Number constraints.Unsigned, Signature comp inner *innerVoterState[Hash, Number, Signature, ID, Environment[Hash, Number, Signature, ID]] finalizedNotifications *wakerChan[finalizedNotification[Hash, Number, Signature, ID]] lastFinalizedNumber Number - globalIn *wakerChan[globalInItem[Hash, Number, Signature, ID]] + globalIn *wakerChan[GlobalInItem[Hash, Number, Signature, ID]] globalOut *buffered[CommunicationOut[Hash, Number, Signature, ID]] // the commit protocol might finalize further than the current round (if we're // behind), we keep track of last finalized in round so we don't violate any @@ -575,7 +554,7 @@ type Voter[Hash constraints.Ordered, Number constraints.Unsigned, Signature comp func NewVoter[Hash constraints.Ordered, Number constraints.Unsigned, Signature comparable, ID constraints.Ordered]( env Environment[Hash, Number, Signature, ID], voters VoterSet[ID], - globalIn chan globalInItem[Hash, Number, Signature, ID], + globalIn chan GlobalInItem[Hash, Number, Signature, ID], globalOutPresend func(CommunicationOut[Hash, Number, Signature, ID]) error, lastRoundNumber uint64, lastRoundVotes []SignedMessage[Hash, Number, Signature, ID], diff --git a/pkg/finality-grandpa/voter_test.go b/pkg/finality-grandpa/voter_test.go index 3ebf5346e6..407630e4c4 100644 --- a/pkg/finality-grandpa/voter_test.go +++ b/pkg/finality-grandpa/voter_test.go @@ -87,7 +87,7 @@ func TestVoter_FinalizingAtFaultThreshold(t *testing.T) { voter := NewVoter[string, uint32, Signature, ID]( &env, *voters, - make(chan globalInItem[string, uint32, Signature, ID]), + make(chan GlobalInItem[string, uint32, Signature, ID]), func(co CommunicationOut[string, uint32, Signature, ID]) error { globalOut <- co; return nil }, 0, nil, @@ -143,7 +143,7 @@ func TestVoter_ExposingVoterState(t *testing.T) { voter := NewVoter[string, uint32, Signature, ID]( &env, *voterSet, - make(chan globalInItem[string, uint32, Signature, ID]), + make(chan GlobalInItem[string, uint32, Signature, ID]), func(co CommunicationOut[string, uint32, Signature, ID]) error { globalOut <- co; return nil }, 0, nil, @@ -229,7 +229,7 @@ func TestVoter_BroadcastCommit(t *testing.T) { voter := NewVoter[string, uint32, Signature, ID]( &env, *voterSet, - make(chan globalInItem[string, uint32, Signature, ID]), + make(chan GlobalInItem[string, uint32, Signature, ID]), func(co CommunicationOut[string, uint32, Signature, ID]) error { globalOut <- co; return nil }, 0, nil, diff --git a/pkg/trie/triedb/cache_bench_test.go b/pkg/trie/triedb/cache_bench_test.go index 6e9b96f302..d337478999 100644 --- a/pkg/trie/triedb/cache_bench_test.go +++ b/pkg/trie/triedb/cache_bench_test.go @@ -3,111 +3,109 @@ package triedb -import ( - "testing" +// import ( +// "testing" - "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" - "github.com/ChainSafe/gossamer/pkg/trie" - "github.com/stretchr/testify/require" -) +// "github.com/ChainSafe/gossamer/internal/primitives/core/hash" +// "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" +// "github.com/ChainSafe/gossamer/pkg/trie" +// "github.com/stretchr/testify/require" +// ) -func Benchmark_ValueCache(b *testing.B) { - entries := map[string][]byte{ - "no": make([]byte, 100), - "noot": make([]byte, 200), - "not": make([]byte, 300), - "notable": make([]byte, 400), - "notification": make([]byte, 500), - "test": make([]byte, 600), - "dimartiro": make([]byte, 700), - } - version := trie.V1 +// func Benchmark_ValueCache(b *testing.B) { +// entries := map[string][]byte{ +// "no": make([]byte, 100), +// "noot": make([]byte, 200), +// "not": make([]byte, 300), +// "notable": make([]byte, 400), +// "notification": make([]byte, 500), +// "test": make([]byte, 600), +// "dimartiro": make([]byte, 700), +// } +// version := trie.V1 - db := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) +// db := NewMemoryDB() +// trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) - for k, v := range entries { - require.NoError(b, trie.Set([]byte(k), v)) - } - err := trie.commit() - require.NoError(b, err) - require.NotEmpty(b, trie.rootHash) - root := trie.rootHash +// for k, v := range entries { +// require.NoError(b, trie.Set([]byte(k), v)) +// } +// err := trie.commit() +// require.NoError(b, err) +// require.NotEmpty(b, trie.rootHash) +// root := trie.rootHash - b.Run("get_value_without_cache", func(b *testing.B) { - trieDB := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Use the deepest key to ensure the trie is traversed fully - val, err := GetWith(trieDB, []byte("notification"), func(d []byte) []byte { return d }) - require.NoError(b, err) - require.NotNil(b, val) - } - }) +// b.Run("get_value_without_cache", func(b *testing.B) { +// trieDB := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// // Use the deepest key to ensure the trie is traversed fully +// val, err := GetWith(trieDB, []byte("notification"), func(d []byte) []byte { return d }) +// require.NoError(b, err) +// require.NotNil(b, val) +// } +// }) - b.Run("get_value_with_cache", func(b *testing.B) { - cache := NewTestTrieCache[hash.H256]() - trieDB := NewTrieDB( - root, db, WithCache[hash.H256, runtime.BlakeTwo256](cache)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Use the deepest key to ensure the trie is traversed fully - val, err := GetWith(trieDB, []byte("notification"), func(d []byte) []byte { return d }) - require.NoError(b, err) - require.NotNil(b, val) - } - }) -} +// b.Run("get_value_with_cache", func(b *testing.B) { +// cache := NewTestTrieCache[hash.H256]() +// trieDB := NewTrieDB( +// root, db, version, WithCache[hash.H256, hasher.Blake2Hasher](cache)) +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// // Use the deepest key to ensure the trie is traversed fully +// val, err := GetWith(trieDB, []byte("notification"), func(d []byte) []byte { return d }) +// require.NoError(b, err) +// require.NotNil(b, val) +// } +// }) +// } -func Benchmark_NodesCache(b *testing.B) { - entries := map[string][]byte{ - "no": make([]byte, 100), - "noot": make([]byte, 200), - "not": make([]byte, 300), - "notable": make([]byte, 400), - "notification": make([]byte, 500), - "test": make([]byte, 600), - "dimartiro": make([]byte, 700), - } - version := trie.V1 +// func Benchmark_NodesCache(b *testing.B) { +// entries := map[string][]byte{ +// "no": make([]byte, 100), +// "noot": make([]byte, 200), +// "not": make([]byte, 300), +// "notable": make([]byte, 400), +// "notification": make([]byte, 500), +// "test": make([]byte, 600), +// "dimartiro": make([]byte, 700), +// } +// version := trie.V1 - db := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) +// db := NewMemoryDB() +// trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) - for k, v := range entries { - require.NoError(b, trie.Set([]byte(k), v)) - } - err := trie.commit() - require.NoError(b, err) - require.NotEmpty(b, trie.rootHash) - root := trie.rootHash +// for k, v := range entries { +// require.NoError(b, trie.Set([]byte(k), v)) +// } +// err := trie.commit() +// require.NoError(b, err) +// require.NotEmpty(b, trie.rootHash) +// root := trie.rootHash - b.Run("iterate_all_entries_without_cache", func(b *testing.B) { - trieDB := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Iterate through all keys - iter, err := NewTrieDBRawIterator(trieDB) - require.NoError(b, err) - for entry, err := iter.NextItem(); entry != nil && err == nil; entry, err = iter.NextItem() { - } - } - }) +// b.Run("iterate_all_entries_without_cache", func(b *testing.B) { +// trieDB := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// // Iterate through all keys +// iter, err := NewTrieDBRawIterator(trieDB) +// require.NoError(b, err) +// for entry, err := iter.NextItem(); entry != nil && err == nil; entry, err = iter.NextItem() { +// } +// } +// }) - // This is the same as iterate_all_entries_without_cache since the raw iterator calls TrieDB.getNodeOrLookup - b.Run("iterate_all_entries_with_cache", func(b *testing.B) { - cache := NewTestTrieCache[hash.H256]() - trieDB := NewTrieDB(root, db, WithCache[hash.H256, runtime.BlakeTwo256](cache)) - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Iterate through all keys - iter, err := NewTrieDBRawIterator(trieDB) - require.NoError(b, err) - for entry, err := iter.NextItem(); entry != nil && err == nil; entry, err = iter.NextItem() { - } - } - }) -} +// // This is the same as iterate_all_entries_without_cache since the raw iterator calls TrieDB.getNodeOrLookup +// b.Run("iterate_all_entries_with_cache", func(b *testing.B) { +// cache := NewTestTrieCache[hash.H256]() +// trieDB := NewTrieDB(root, db, version, WithCache[hash.H256, hasher.Blake2Hasher](cache)) +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// // Iterate through all keys +// iter, err := NewTrieDBRawIterator(trieDB) +// require.NoError(b, err) +// for entry, err := iter.NextItem(); entry != nil && err == nil; entry, err = iter.NextItem() { +// } +// } +// }) +// } diff --git a/pkg/trie/triedb/cache_test.go b/pkg/trie/triedb/cache_test.go index 49e8c6ef5b..b0d24fcc36 100644 --- a/pkg/trie/triedb/cache_test.go +++ b/pkg/trie/triedb/cache_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/ChainSafe/gossamer/pkg/trie/triedb/codec" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibbles" "github.com/stretchr/testify/assert" @@ -16,7 +16,7 @@ import ( ) func Test_ByteSize(t *testing.T) { - var childHash hash.H256 = runtime.BlakeTwo256{}.Hash([]byte{0}) + var childHash hash.H256 = hasher.Blake2Hasher{}.Hash([]byte{0}) encodedBranch := codec.Branch{ PartialKey: nibbles.NewNibbles([]byte{1}), Value: codec.InlineValue([]byte{7, 8, 9}), @@ -27,7 +27,7 @@ func Test_ByteSize(t *testing.T) { }, } - cachedNode, err := NewCachedNodeFromNode[hash.H256, runtime.BlakeTwo256](encodedBranch) + cachedNode, err := NewCachedNodeFromNode[hash.H256, hasher.Blake2Hasher](encodedBranch) require.NoError(t, err) assert.Equal(t, 308, int(cachedNode.ByteSize())) @@ -40,7 +40,7 @@ func Test_ByteSize(t *testing.T) { codec.HashedNode[hash.H256]{Hash: childHash}, }, } - cachedNode, err = NewCachedNodeFromNode[hash.H256, runtime.BlakeTwo256](encodedBranch) + cachedNode, err = NewCachedNodeFromNode[hash.H256, hasher.Blake2Hasher](encodedBranch) require.NoError(t, err) assert.Equal(t, 308+30, int(cachedNode.ByteSize())) } diff --git a/pkg/trie/triedb/codec/decode_test.go b/pkg/trie/triedb/codec/decode_test.go index eacb09ed9c..285dd41b6f 100644 --- a/pkg/trie/triedb/codec/decode_test.go +++ b/pkg/trie/triedb/codec/decode_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibbles" "github.com/stretchr/testify/assert" @@ -29,7 +29,7 @@ func scaleEncodeByteSlice(t *testing.T, b []byte) (encoded []byte) { func Test_Decode(t *testing.T) { t.Parallel() - hashedValue := runtime.BlakeTwo256{}.Hash([]byte("test")) + hashedValue := hasher.Blake2Hasher{}.Hash([]byte("test")) testCases := map[string]struct { reader io.Reader @@ -153,7 +153,7 @@ func Test_Decode(t *testing.T) { func Test_decodeBranch(t *testing.T) { t.Parallel() - var childHash hash.H256 = runtime.BlakeTwo256{}.Hash([]byte{0}) + var childHash hash.H256 = hasher.Blake2Hasher{}.Hash([]byte{0}) scaleEncodedChildHash := scaleEncodeByteSlice(t, childHash.Bytes()) testCases := map[string]struct { diff --git a/pkg/trie/triedb/iterator_test.go b/pkg/trie/triedb/iterator_test.go index 31b342c950..befe8f2aa7 100644 --- a/pkg/trie/triedb/iterator_test.go +++ b/pkg/trie/triedb/iterator_test.go @@ -3,15 +3,31 @@ package triedb import ( + "math" "testing" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" - "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type ( + // / substrate trie layout + layoutV0 struct{} + // / substrate trie layout, with external value nodes. + layoutV1 struct{} +) + +func (l layoutV0) MaxInlineValue() int { + return math.MaxInt +} +func (l layoutV1) MaxInlineValue() int { + return 32 +} + +var layout = layoutV1{} + func Test_TrieDBRawIterator(t *testing.T) { entries := map[string][]byte{ "no": make([]byte, 1), @@ -26,8 +42,7 @@ func Test_TrieDBRawIterator(t *testing.T) { } db := NewMemoryDB() - trieDB := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trieDB.SetVersion(trie.V1) + trieDB := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, layout) for k, v := range entries { err := trieDB.Set([]byte(k), v) @@ -181,8 +196,7 @@ func TestTrieDBIterator(t *testing.T) { } db := NewMemoryDB() - trieDB := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trieDB.SetVersion(trie.V1) + trieDB := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, layout) for k, v := range entries { err := trieDB.Set([]byte(k), v) @@ -279,8 +293,7 @@ func TestTrieDBKeyIterator(t *testing.T) { } db := NewMemoryDB() - trieDB := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trieDB.SetVersion(trie.V1) + trieDB := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, layout) for k, v := range entries { err := trieDB.Set([]byte(k), v) diff --git a/pkg/trie/triedb/lookup.go b/pkg/trie/triedb/lookup.go index 82c5dca1cc..3a1a58db02 100644 --- a/pkg/trie/triedb/lookup.go +++ b/pkg/trie/triedb/lookup.go @@ -9,7 +9,6 @@ import ( "slices" hashdb "github.com/ChainSafe/gossamer/internal/hash-db" - "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/pkg/trie/triedb/codec" "github.com/ChainSafe/gossamer/pkg/trie/triedb/hash" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibbles" @@ -29,7 +28,7 @@ type TrieLookup[H hash.Hash, Hasher hash.Hasher[H], QueryItem any] struct { // optional recorder for recording trie accesses recorder TrieRecorder // layout for the trie - layout trie.TrieLayout + layout TrieLayout // query function query Query[QueryItem] } @@ -40,6 +39,7 @@ func NewTrieLookup[H hash.Hash, Hasher hash.Hasher[H], QueryItem any]( hash H, cache TrieCache[H], recorder TrieRecorder, + layout TrieLayout, query Query[QueryItem], ) TrieLookup[H, Hasher, QueryItem] { return TrieLookup[H, Hasher, QueryItem]{ @@ -47,6 +47,7 @@ func NewTrieLookup[H hash.Hash, Hasher hash.Hasher[H], QueryItem any]( hash: hash, cache: cache, recorder: recorder, + layout: layout, query: query, } } diff --git a/pkg/trie/triedb/lookup_test.go b/pkg/trie/triedb/lookup_test.go index 8bfdb6040b..af212783dc 100644 --- a/pkg/trie/triedb/lookup_test.go +++ b/pkg/trie/triedb/lookup_test.go @@ -8,8 +8,7 @@ import ( memorydb "github.com/ChainSafe/gossamer/internal/memory-db" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" - "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibbles" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,10 +17,10 @@ import ( func TestTrieDB_Lookup(t *testing.T) { t.Run("root_not_exists_in_db", func(t *testing.T) { db := memorydb.NewMemoryDB[ - hash.H256, runtime.BlakeTwo256, hash.H256, memorydb.HashKey[hash.H256], + hash.H256, hasher.Blake2Hasher, hash.H256, memorydb.HashKey[hash.H256], ]([]byte("not0")) - empty := runtime.BlakeTwo256{}.Hash([]byte{0}) - lookup := NewTrieLookup[hash.H256, runtime.BlakeTwo256, []byte](&db, empty, nil, nil, nil) + empty := hasher.Blake2Hasher{}.Hash([]byte{0}) + lookup := NewTrieLookup[hash.H256, hasher.Blake2Hasher, []byte](&db, empty, nil, nil, layoutV0{}, nil) value, err := lookup.Lookup([]byte("test")) assert.Nil(t, value) @@ -43,11 +42,10 @@ func (*trieCacheImpl) GetNode(hash hash.H256) CachedNode[hash.H256] { return nil func Test_TrieLookup_lookupValueWithCache(t *testing.T) { cache := &trieCacheImpl{} inmemoryDB := NewMemoryDB() - trieDB := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256]( - inmemoryDB, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trieDB := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher]( + inmemoryDB, layoutV1{}, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trieDB.SetVersion(trie.V1) entries := map[string][]byte{ "no": make([]byte, 1), @@ -66,11 +64,12 @@ func Test_TrieLookup_lookupValueWithCache(t *testing.T) { err := trieDB.commit() require.NoError(t, err) - lookup := NewTrieLookup[hash.H256, runtime.BlakeTwo256]( + lookup := NewTrieLookup[hash.H256, hasher.Blake2Hasher]( inmemoryDB, trieDB.rootHash, cache, nil, + layoutV1{}, func(data []byte) []byte { return data }, diff --git a/pkg/trie/triedb/proof/generate_test.go b/pkg/trie/triedb/proof/generate_test.go index ce1b323ef4..48c3af1e32 100644 --- a/pkg/trie/triedb/proof/generate_test.go +++ b/pkg/trie/triedb/proof/generate_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/pkg/trie/triedb" "github.com/stretchr/testify/assert" @@ -19,7 +19,7 @@ func Test_NewProof(t *testing.T) { entries []trie.Entry storageVersion trie.TrieLayout keys []string - expectedProof MerkleProof[hash.H256, runtime.BlakeTwo256] + expectedProof MerkleProof[hash.H256, hasher.Blake2Hasher] }{ "leaf": { entries: []trie.Entry{ @@ -29,7 +29,7 @@ func Test_NewProof(t *testing.T) { }, }, keys: []string{"a"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ {66, 97, 0}, // 'a' node without value }, }, @@ -45,7 +45,7 @@ func Test_NewProof(t *testing.T) { }, }, keys: []string{"ab"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ {194, 97, 64, 0, 4, 97, 12, 65, 2, 0}, }, }, @@ -77,7 +77,7 @@ func Test_NewProof(t *testing.T) { }, }, keys: []string{"go"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ { 128, 192, 0, 0, 128, 114, 166, 121, 79, 225, 146, 229, 34, 68, 211, 54, 148, 205, 192, 58, 131, 95, 46, 239, @@ -118,7 +118,7 @@ func Test_NewProof(t *testing.T) { }, }, keys: []string{"go", "polkadot"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ { 128, 192, 0, 0, 0, }, @@ -141,7 +141,7 @@ func Test_NewProof(t *testing.T) { t.Run(name, func(t *testing.T) { // Build trie inmemoryDB := NewMemoryDB() - triedb := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + triedb := triedb.NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, trie.V0) for _, entry := range testCase.entries { triedb.Set(entry.Key, entry.Value) @@ -150,7 +150,7 @@ func Test_NewProof(t *testing.T) { root := triedb.MustHash() // Generate proof - proof, err := NewMerkleProof[hash.H256, runtime.BlakeTwo256]( + proof, err := NewMerkleProof[hash.H256, hasher.Blake2Hasher]( inmemoryDB, testCase.storageVersion, root, testCase.keys) require.NoError(t, err) assert.Equal(t, len(testCase.expectedProof), len(proof)) diff --git a/pkg/trie/triedb/proof/proof.go b/pkg/trie/triedb/proof/proof.go index 80fd7c8d2c..8f2468446d 100644 --- a/pkg/trie/triedb/proof/proof.go +++ b/pkg/trie/triedb/proof/proof.go @@ -42,8 +42,7 @@ func NewMerkleProof[H hash.Hash, Hasher hash.Hasher[H]]( // Traverse the trie recording the visited nodes recorder := triedb.NewRecorder[H]() - trie := triedb.NewTrieDB[H, Hasher](rootHash, db, triedb.WithRecorder[H, Hasher](recorder)) - trie.SetVersion(trieVersion) + trie := triedb.NewTrieDB[H, Hasher](rootHash, db, trieVersion, triedb.WithRecorder[H, Hasher](recorder)) _, err = trie.Get(key) if err != nil { return nil, err diff --git a/pkg/trie/triedb/proof/proof_test.go b/pkg/trie/triedb/proof/proof_test.go index 7fc295bb89..4b667009f2 100644 --- a/pkg/trie/triedb/proof/proof_test.go +++ b/pkg/trie/triedb/proof/proof_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/ChainSafe/gossamer/pkg/trie" "github.com/ChainSafe/gossamer/pkg/trie/triedb" @@ -19,7 +19,7 @@ func Test_GenerateAndVerify(t *testing.T) { testCases := map[string]struct { entries []trie.Entry keys []string - expectedProof MerkleProof[hash.H256, runtime.BlakeTwo256] + expectedProof MerkleProof[hash.H256, hasher.Blake2Hasher] }{ "leaf": { entries: []trie.Entry{ @@ -29,7 +29,7 @@ func Test_GenerateAndVerify(t *testing.T) { }, }, keys: []string{"a"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ {66, 97, 0}, // 'a' node without value }, }, @@ -45,7 +45,7 @@ func Test_GenerateAndVerify(t *testing.T) { }, }, keys: []string{"ab"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ {194, 97, 64, 0, 4, 97, 12, 65, 2, 0}, }, }, @@ -77,7 +77,7 @@ func Test_GenerateAndVerify(t *testing.T) { }, }, keys: []string{"go"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ { 128, 192, 0, 0, 128, 114, 166, 121, 79, 225, 146, 229, 34, 68, 211, 54, 148, 205, 192, 58, 131, 95, 46, 239, @@ -118,7 +118,7 @@ func Test_GenerateAndVerify(t *testing.T) { }, }, keys: []string{"go", "polkadot"}, - expectedProof: MerkleProof[hash.H256, runtime.BlakeTwo256]{ + expectedProof: MerkleProof[hash.H256, hasher.Blake2Hasher]{ { 128, 192, 0, 0, 0, }, @@ -144,8 +144,7 @@ func Test_GenerateAndVerify(t *testing.T) { t.Run(fmt.Sprintf("%s_%s", name, trieVersion.String()), func(t *testing.T) { // Build trie inmemoryDB := NewMemoryDB() - triedb := triedb.NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) - triedb.SetVersion(trieVersion) + triedb := triedb.NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, trieVersion) for _, entry := range testCase.entries { triedb.Set(entry.Key, entry.Value) @@ -154,7 +153,7 @@ func Test_GenerateAndVerify(t *testing.T) { root := triedb.MustHash() // Generate proof - proof, err := NewMerkleProof[hash.H256, runtime.BlakeTwo256](inmemoryDB, trieVersion, root, testCase.keys) + proof, err := NewMerkleProof[hash.H256, hasher.Blake2Hasher](inmemoryDB, trieVersion, root, testCase.keys) require.NoError(t, err) require.Equal(t, testCase.expectedProof, proof) diff --git a/pkg/trie/triedb/proof/util_test.go b/pkg/trie/triedb/proof/util_test.go index 0e41363f1c..b5fb23a3ae 100644 --- a/pkg/trie/triedb/proof/util_test.go +++ b/pkg/trie/triedb/proof/util_test.go @@ -6,13 +6,13 @@ package proof import ( memorydb "github.com/ChainSafe/gossamer/internal/memory-db" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" ) func NewMemoryDB() *memorydb.MemoryDB[ - hash.H256, runtime.BlakeTwo256, hash.H256, memorydb.HashKey[hash.H256]] { + hash.H256, hasher.Blake2Hasher, hash.H256, memorydb.HashKey[hash.H256]] { db := memorydb.NewMemoryDB[ - hash.H256, runtime.BlakeTwo256, hash.H256, memorydb.HashKey[hash.H256], + hash.H256, hasher.Blake2Hasher, hash.H256, memorydb.HashKey[hash.H256], ]([]byte{0}) return &db } diff --git a/pkg/trie/triedb/recorder_test.go b/pkg/trie/triedb/recorder_test.go index 5ac62b8f2d..650d56888d 100644 --- a/pkg/trie/triedb/recorder_test.go +++ b/pkg/trie/triedb/recorder_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/stretchr/testify/require" ) @@ -15,7 +15,7 @@ import ( // https://github.com/dimartiro/substrate-trie-test/blob/master/src/substrate_trie_test.rs func TestRecorder(t *testing.T) { inmemoryDB := NewMemoryDB() - triedb := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + triedb := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) triedb.Set([]byte("pol"), []byte("polvalue")) triedb.Set([]byte("polka"), []byte("polkavalue")) @@ -30,7 +30,7 @@ func TestRecorder(t *testing.T) { t.Run("Record_pol_access_should_record_2_node", func(t *testing.T) { recorder := NewRecorder[hash.H256]() trie := NewTrieDB( - root, inmemoryDB, WithRecorder[hash.H256, runtime.BlakeTwo256](recorder)) + root, inmemoryDB, layoutV0{}, WithRecorder[hash.H256, hasher.Blake2Hasher](recorder)) trie.Get([]byte("pol")) @@ -66,8 +66,8 @@ func TestRecorder(t *testing.T) { t.Run("Record_go_access_should_record_2_nodes", func(t *testing.T) { recorder := NewRecorder[hash.H256]() - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256]( - root, inmemoryDB, WithRecorder[hash.H256, runtime.BlakeTwo256](recorder)) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher]( + root, inmemoryDB, layoutV0{}, WithRecorder[hash.H256, hasher.Blake2Hasher](recorder)) trie.Get([]byte("go")) diff --git a/pkg/trie/triedb/triedb.go b/pkg/trie/triedb/triedb.go index 7e7314ab8f..ec5e721aaa 100644 --- a/pkg/trie/triedb/triedb.go +++ b/pkg/trie/triedb/triedb.go @@ -9,8 +9,6 @@ import ( "fmt" "slices" - "github.com/ChainSafe/gossamer/pkg/trie" - hashdb "github.com/ChainSafe/gossamer/internal/hash-db" "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/pkg/trie/triedb/codec" @@ -40,19 +38,36 @@ func WithRecorder[H hash.Hash, Hasher hash.Hasher[H]](r TrieRecorder) TrieDBOpts } } -type TrieLayout = trie.TrieLayout - -var ( - V0 = trie.V0 - V1 = trie.V1 -) +/// Trait with definition of trie layout. +/// Contains all associated trait needed for +/// a trie definition or implementation. +// pub trait TrieLayout { +// /// If true, the trie will use extension nodes and +// /// no partial in branch, if false the trie will only +// /// use branch and node with partials in both. +// const USE_EXTENSION: bool; +// /// If true, the trie will allow empty values into `TrieDBMut` +// const ALLOW_EMPTY: bool = false; +// /// Threshold above which an external node should be +// /// use to store a node value. +// const MAX_INLINE_VALUE: Option; + +// /// Hasher to use for this trie. +// type Hash: Hasher; +// /// Codec to use (needs to match hasher and nibble ops). +// type Codec: NodeCodec::Out>; +// } + +type TrieLayout interface { + MaxInlineValue() int +} // TrieDB is a DB-backed patricia merkle trie implementation // using lazy loading to fetch nodes type TrieDB[H hash.Hash, Hasher hash.Hasher[H]] struct { rootHash H db hashdb.HashDB[H] - version trie.TrieLayout + layout TrieLayout // rootHandle is an in-memory-trie-like representation of the node // references and new inserted nodes in the trie rootHandle NodeHandle @@ -69,10 +84,11 @@ type TrieDB[H hash.Hash, Hasher hash.Hasher[H]] struct { } func NewEmptyTrieDB[H hash.Hash, Hasher hash.Hasher[H]]( - db hashdb.HashDB[H], opts ...TrieDBOpts[H, Hasher]) *TrieDB[H, Hasher] { + db hashdb.HashDB[H], layout TrieLayout, opts ...TrieDBOpts[H, Hasher], +) *TrieDB[H, Hasher] { hasher := *new(Hasher) root := hasher.Hash([]byte{0}) - return NewTrieDB[H, Hasher](root, db, opts...) + return NewTrieDB[H, Hasher](root, db, layout, opts...) } type hashPrefix[H hash.Hash] struct { @@ -82,12 +98,16 @@ type hashPrefix[H hash.Hash] struct { // NewTrieDB creates a new TrieDB using the given root and db func NewTrieDB[H hash.Hash, Hasher hash.Hasher[H]]( - rootHash H, db hashdb.HashDB[H], opts ...TrieDBOpts[H, Hasher]) *TrieDB[H, Hasher] { + rootHash H, + db hashdb.HashDB[H], + layout TrieLayout, + opts ...TrieDBOpts[H, Hasher], +) *TrieDB[H, Hasher] { rootHandle := persisted[H]{rootHash} trieDB := &TrieDB[H, Hasher]{ rootHash: rootHash, - version: trie.V0, + layout: layout, db: db, storage: newNodeStorage[H](), rootHandle: rootHandle, @@ -101,14 +121,6 @@ func NewTrieDB[H hash.Hash, Hasher hash.Hasher[H]]( return trieDB } -func (t *TrieDB[H, Hasher]) SetVersion(v TrieLayout) { - if v < t.version { - panic("cannot regress trie version") - } - - t.version = v -} - // Hash returns the hashed root of the trie. func (t *TrieDB[H, Hasher]) Hash() (H, error) { err := t.commit() @@ -152,6 +164,7 @@ func (t *TrieDB[H, Hasher]) lookup(fullKey []byte, handle NodeHandle) ([]byte, e node.hash, nil, // no cache intentionally t.recorder, + t.layout, func(data []byte) []byte { return data }, @@ -633,7 +646,7 @@ func (t *TrieDB[H, Hasher]) insertInspector( case Empty: // If the node is empty we have to replace it with a leaf node with the // new value - value := NewValue[H](value, t.version.MaxInlineValue()) + value := NewValue[H](value, t.layout.MaxInlineValue()) pnk := partial.NodeKey() return replaceNode{node: Leaf[H]{partialKey: pnk, value: value}}, nil case Leaf[H]: @@ -643,7 +656,7 @@ func (t *TrieDB[H, Hasher]) insertInspector( if common == existingKey.Len() && common == partial.Len() { // We are trying to insert a value in the same leaf so we just need // to replace the value - value := NewValue[H](value, t.version.MaxInlineValue()) + value := NewValue[H](value, t.layout.MaxInlineValue()) unchanged := n.value.equal(value) keyVal := keyNibbles.Clone() keyVal.Advance(existingKey.Len()) @@ -702,7 +715,7 @@ func (t *TrieDB[H, Hasher]) insertInspector( if common == existingKey.Len() && common == partial.Len() { // We are trying to insert a value in the same branch so we just need // to replace the value - value := NewValue[H](value, t.version.MaxInlineValue()) + value := NewValue[H](value, t.layout.MaxInlineValue()) var unchanged bool if n.value != nil { unchanged = n.value.equal(value) @@ -732,7 +745,7 @@ func (t *TrieDB[H, Hasher]) insertInspector( ix := existingKey.At(common) children[ix] = inMemory(allocStorage) - value := NewValue[H](value, t.version.MaxInlineValue()) + value := NewValue[H](value, t.layout.MaxInlineValue()) if partial.Len()-common == 0 { // The value should be part of the branch @@ -783,7 +796,7 @@ func (t *TrieDB[H, Hasher]) insertInspector( } } else { // Original has nothing here so we have to create a new leaf - value := NewValue[H](value, t.version.MaxInlineValue()) + value := NewValue[H](value, t.layout.MaxInlineValue()) leaf := t.storage.alloc(NewStoredNode{node: Leaf[H]{keyNibbles.NodeKey(), value}}) n.children[idx] = inMemory(leaf) } @@ -1134,7 +1147,7 @@ func (t *TrieDB[H, Hasher]) recordAccess(access TrieAccess) { func (t *TrieDB[H, Hasher]) GetHash(key []byte) (*H, error) { // TODO: look into moving query into Lookup method lookup := NewTrieLookup[H, Hasher]( - t.db, t.rootHash, t.cache, t.recorder, + t.db, t.rootHash, t.cache, t.recorder, t.layout, func([]byte) any { return nil }, ) return lookup.LookupHash(key) @@ -1145,14 +1158,14 @@ func GetWith[H hash.Hash, Hasher hash.Hasher[H], QueryItem any]( t *TrieDB[H, Hasher], key []byte, query Query[QueryItem], ) (*QueryItem, error) { lookup := NewTrieLookup[H, Hasher]( - t.db, t.rootHash, t.cache, t.recorder, query, + t.db, t.rootHash, t.cache, t.recorder, t.layout, query, ) return lookup.Lookup(key) } func (t *TrieDB[H, Hasher]) LookupFirstDescendant(key []byte) (MerkleValue[H], error) { lookup := NewTrieLookup[H, Hasher]( - t.db, t.rootHash, t.cache, t.recorder, func([]byte) any { return nil }, + t.db, t.rootHash, t.cache, t.recorder, t.layout, func([]byte) any { return nil }, ) return lookup.LookupFirstDescendant(key, nibbles.NewNibbles(slices.Clone(key))) } diff --git a/pkg/trie/triedb/triedb_iterator_test.go b/pkg/trie/triedb/triedb_iterator_test.go index c39a0c3ea4..76ba424a37 100644 --- a/pkg/trie/triedb/triedb_iterator_test.go +++ b/pkg/trie/triedb/triedb_iterator_test.go @@ -3,95 +3,95 @@ package triedb -import ( - "testing" - - "github.com/ChainSafe/gossamer/internal/database" - "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" - "github.com/ChainSafe/gossamer/pkg/trie" - "github.com/ChainSafe/gossamer/pkg/trie/inmemory" - "github.com/stretchr/testify/assert" -) - -func newTestDB(t assert.TestingT) database.Table { - db, err := database.NewPebble("", true) - assert.NoError(t, err) - return database.NewTable(db, "trie") -} - -func TestIterator(t *testing.T) { - db := newTestDB(t) - inMemoryTrie := inmemory.NewEmptyTrie() - inMemoryTrie.SetVersion(trie.V1) - - entries := map[string][]byte{ - "no": make([]byte, 1), - "noot": make([]byte, 2), - "not": make([]byte, 3), - "notable": make([]byte, 4), - "notification": make([]byte, 5), - "test": make([]byte, 6), - "dimartiro": make([]byte, 7), - } - - for k, v := range entries { - inMemoryTrie.Put([]byte(k), v) - } - err := inMemoryTrie.WriteDirty(db) - assert.NoError(t, err) - - root, err := inMemoryTrie.Hash() - assert.NoError(t, err) - - inmemoryDB := NewMemoryDB() - trieDB := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) - - for k, v := range entries { - err := trieDB.Set([]byte(k), v) - assert.NoError(t, err) - } - assert.NoError(t, trieDB.commit()) - - // check that the root hashes are the same - assert.Equal(t, root.ToBytes(), trieDB.rootHash.Bytes()) - - t.Run("iterate_over_all_entries", func(t *testing.T) { - iter, err := NewTrieDBRawIterator(trieDB) - assert.NoError(t, err) - - expected := inMemoryTrie.NextKey([]byte{}) - i := 0 - for { - item, err := iter.NextItem() - assert.NoError(t, err) - if item == nil { - break - } - assert.Equal(t, expected, item.Key) - expected = inMemoryTrie.NextKey(expected) - i++ - } - assert.Equal(t, len(entries), i) - }) - - t.Run("iterate_after_seeking", func(t *testing.T) { - iter, err := NewTrieDBRawIterator(trieDB) - assert.NoError(t, err) - - found, err := iter.seek([]byte("not"), true) - assert.NoError(t, err) - assert.True(t, found) - - expected := inMemoryTrie.NextKey([]byte("not")) - actual, err := iter.NextItem() - assert.NoError(t, err) - assert.NotNil(t, actual) - - assert.Equal(t, []byte("not"), actual.Key) - actual, err = iter.NextItem() - assert.NoError(t, err) - assert.NotNil(t, actual) - assert.Equal(t, expected, actual.Key) - }) -} +// import ( +// "testing" + +// "github.com/ChainSafe/gossamer/internal/database" +// "github.com/ChainSafe/gossamer/internal/primitives/core/hash" +// "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" +// "github.com/ChainSafe/gossamer/pkg/trie" +// "github.com/ChainSafe/gossamer/pkg/trie/inmemory" +// "github.com/stretchr/testify/assert" +// ) + +// func newTestDB(t assert.TestingT) database.Table { +// db, err := database.NewPebble("", true) +// assert.NoError(t, err) +// return database.NewTable(db, "trie") +// } + +// func TestIterator(t *testing.T) { +// db := newTestDB(t) +// inMemoryTrie := inmemory.NewEmptyTrie() +// inMemoryTrie.SetVersion(trie.V1) + +// entries := map[string][]byte{ +// "no": make([]byte, 1), +// "noot": make([]byte, 2), +// "not": make([]byte, 3), +// "notable": make([]byte, 4), +// "notification": make([]byte, 5), +// "test": make([]byte, 6), +// "dimartiro": make([]byte, 7), +// } + +// for k, v := range entries { +// inMemoryTrie.Put([]byte(k), v) +// } +// err := inMemoryTrie.WriteDirty(db) +// assert.NoError(t, err) + +// root, err := inMemoryTrie.Hash() +// assert.NoError(t, err) + +// inmemoryDB := NewMemoryDB() +// trieDB := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) + +// for k, v := range entries { +// err := trieDB.Set([]byte(k), v) +// assert.NoError(t, err) +// } +// assert.NoError(t, trieDB.commit()) + +// // check that the root hashes are the same +// assert.Equal(t, root.ToBytes(), trieDB.rootHash.Bytes()) + +// t.Run("iterate_over_all_entries", func(t *testing.T) { +// iter, err := NewTrieDBRawIterator(trieDB) +// assert.NoError(t, err) + +// expected := inMemoryTrie.NextKey([]byte{}) +// i := 0 +// for { +// item, err := iter.NextItem() +// assert.NoError(t, err) +// if item == nil { +// break +// } +// assert.Equal(t, expected, item.Key) +// expected = inMemoryTrie.NextKey(expected) +// i++ +// } +// assert.Equal(t, len(entries), i) +// }) + +// t.Run("iterate_after_seeking", func(t *testing.T) { +// iter, err := NewTrieDBRawIterator(trieDB) +// assert.NoError(t, err) + +// found, err := iter.seek([]byte("not"), true) +// assert.NoError(t, err) +// assert.True(t, found) + +// expected := inMemoryTrie.NextKey([]byte("not")) +// actual, err := iter.NextItem() +// assert.NoError(t, err) +// assert.NotNil(t, actual) + +// assert.Equal(t, []byte("not"), actual.Key) +// actual, err = iter.NextItem() +// assert.NoError(t, err) +// assert.NotNil(t, actual) +// assert.Equal(t, expected, actual.Key) +// }) +// } diff --git a/pkg/trie/triedb/triedb_test.go b/pkg/trie/triedb/triedb_test.go index ff7a2f9262..a04b4d6938 100644 --- a/pkg/trie/triedb/triedb_test.go +++ b/pkg/trie/triedb/triedb_test.go @@ -5,13 +5,14 @@ package triedb import ( "bytes" + "fmt" "slices" "testing" hashdb "github.com/ChainSafe/gossamer/internal/hash-db" "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" - "github.com/ChainSafe/gossamer/pkg/trie" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" + "github.com/ChainSafe/gossamer/internal/primitives/kv" "github.com/ChainSafe/gossamer/pkg/trie/triedb/codec" "github.com/ChainSafe/gossamer/pkg/trie/triedb/nibbles" "github.com/stretchr/testify/assert" @@ -22,14 +23,14 @@ func TestInsertions(t *testing.T) { t.Parallel() testCases := map[string]struct { - trieEntries []trie.Entry + trieEntries []kv.KeyValue key []uint8 value []uint8 stored nodeStorage[hash.H256] dontCheck bool }{ "nil_parent": { - trieEntries: []trie.Entry{}, + trieEntries: []kv.KeyValue{}, key: []byte{0x01}, value: []byte("leaf"), stored: nodeStorage[hash.H256]{ @@ -44,7 +45,7 @@ func TestInsertions(t *testing.T) { }, }, "branch_parent": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{0x01}, Value: []byte("branch"), @@ -74,7 +75,7 @@ func TestInsertions(t *testing.T) { }, }, "branch_in_between_rearrange": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("branch"), @@ -118,7 +119,7 @@ func TestInsertions(t *testing.T) { }, }, "branch_in_between": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1, 0}, Value: []byte("branch"), @@ -162,7 +163,7 @@ func TestInsertions(t *testing.T) { }, }, "override_branch_value": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("branch"), @@ -197,7 +198,7 @@ func TestInsertions(t *testing.T) { dontCheck: true, }, "override_branch_value_same_value": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("branch"), @@ -231,7 +232,7 @@ func TestInsertions(t *testing.T) { }, }, "override_leaf_of_branch_value_same_value": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("branch"), @@ -265,7 +266,7 @@ func TestInsertions(t *testing.T) { }, }, "override_leaf_parent": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("leaf"), @@ -286,7 +287,7 @@ func TestInsertions(t *testing.T) { dontCheck: true, }, "write_same_leaf_value_to_leaf_parent": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("same"), @@ -306,7 +307,7 @@ func TestInsertions(t *testing.T) { }, }, "write_leaf_as_divergent_child_next_to_parent_leaf": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{0x01, 0x02}, Value: []byte("original leaf"), @@ -354,7 +355,7 @@ func TestInsertions(t *testing.T) { t.Parallel() // Setup trie inmemoryDB := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) for _, entry := range testCase.trieEntries { require.NoError(t, trie.Set(entry.Key, entry.Value)) @@ -385,12 +386,12 @@ func TestDeletes(t *testing.T) { t.Parallel() testCases := map[string]struct { - trieEntries []trie.Entry + trieEntries []kv.KeyValue key []byte expected nodeStorage[hash.H256] }{ "nil_key": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("leaf"), @@ -414,7 +415,7 @@ func TestDeletes(t *testing.T) { }, }, "delete_leaf": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("leaf"), @@ -426,7 +427,7 @@ func TestDeletes(t *testing.T) { }, }, "delete_branch": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("branch"), @@ -450,7 +451,7 @@ func TestDeletes(t *testing.T) { }, }, "delete_branch_without_value_should_do_nothing": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1, 0}, Value: []byte("leaf1"), @@ -495,7 +496,7 @@ func TestDeletes(t *testing.T) { // Setup trie inmemoryDB := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) for _, entry := range testCase.trieEntries { assert.NoError(t, trie.Set(entry.Key, entry.Value)) @@ -515,13 +516,13 @@ func TestInsertAfterDelete(t *testing.T) { t.Parallel() testCases := map[string]struct { - trieEntries []trie.Entry + trieEntries []kv.KeyValue key []byte value []byte expected nodeStorage[hash.H256] }{ "insert_leaf_after_delete": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("leaf"), @@ -541,7 +542,7 @@ func TestInsertAfterDelete(t *testing.T) { }, }, "insert_branch_after_delete": { - trieEntries: []trie.Entry{ + trieEntries: []kv.KeyValue{ { Key: []byte{1}, Value: []byte("branch"), @@ -582,7 +583,7 @@ func TestInsertAfterDelete(t *testing.T) { // Setup trie inmemoryDB := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) for _, entry := range testCase.trieEntries { assert.NoError(t, trie.insert(nibbles.NewNibbles(entry.Key), entry.Value)) @@ -609,7 +610,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) err := trie.Set([]byte("leaf"), []byte("leafvalue")) assert.NoError(t, err) @@ -629,7 +630,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) err := trie.Set([]byte("branchleaf"), []byte("leafvalue")) assert.NoError(t, err) @@ -653,7 +654,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - tr := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + tr := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) err := tr.Set([]byte("branchleaf"), make([]byte, 40)) assert.NoError(t, err) @@ -678,8 +679,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - tr := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) - tr.SetVersion(trie.V1) + tr := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV1{}) err := tr.Set([]byte("leaf"), make([]byte, 40)) assert.NoError(t, err) @@ -700,8 +700,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - tr := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) - tr.SetVersion(trie.V1) + tr := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV1{}) err := tr.Set([]byte("leaf"), make([]byte, 40)) assert.NoError(t, err) @@ -724,8 +723,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - tr := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) - tr.SetVersion(trie.V1) + tr := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV1{}) err := tr.Set([]byte("branchleaf"), make([]byte, 40)) assert.NoError(t, err) @@ -751,8 +749,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - tr := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) - tr.SetVersion(trie.V1) + tr := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV1{}) err := tr.Set([]byte("branchleaf"), make([]byte, 40)) assert.NoError(t, err) @@ -781,7 +778,7 @@ func TestDBCommits(t *testing.T) { t.Parallel() inmemoryDB := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](inmemoryDB) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](inmemoryDB, layoutV0{}) err := trie.Set([]byte("branchleaf"), []byte("leafvalue")) assert.NoError(t, err) @@ -810,8 +807,8 @@ func TestDBCommits(t *testing.T) { func Test_TrieDB(t *testing.T) { t.Run("recorder", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -824,8 +821,7 @@ func Test_TrieDB(t *testing.T) { // Add some initial data to the trie db := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues[:1] { require.NoError(t, trie.Set(entry.key, entry.value)) @@ -841,10 +837,9 @@ func Test_TrieDB(t *testing.T) { overlay := db.Clone() newRoot := root { - trie := NewTrieDB(newRoot, &overlay, - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB(newRoot, &overlay, version, + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) for _, entry := range keyValues[1:] { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -856,7 +851,7 @@ func Test_TrieDB(t *testing.T) { partialDB := NewMemoryDB() for _, record := range recorder.Drain() { - // key := runtime.BlakeTwo256{}.Hash(record.Data).Bytes() + // key := hasher.Blake2Hasher{}.Hash(record.Data).Bytes() // require.NoError(t, partialDB.Set(key, record.Data)) partialDB.Insert(hashdb.EmptyPrefix, record.Data) } @@ -864,8 +859,7 @@ func Test_TrieDB(t *testing.T) { // Replay the it, but this time we use the proof. var validatedRoot hash.H256 { - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, partialDB) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, partialDB, version) for _, entry := range keyValues[1:] { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -880,8 +874,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("recorder_with_cache", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -894,8 +888,7 @@ func Test_TrieDB(t *testing.T) { // Add some initial data to the trie db := NewMemoryDB() - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues[:1] { require.NoError(t, trie.Set(entry.key, entry.value)) @@ -908,8 +901,7 @@ func Test_TrieDB(t *testing.T) { cache := NewTestTrieCache[hash.H256]() { - trie := NewTrieDB(trie.rootHash, db, WithCache[hash.H256, runtime.BlakeTwo256](cache)) - trie.SetVersion(version) + trie := NewTrieDB(trie.rootHash, db, version, WithCache[hash.H256, hasher.Blake2Hasher](cache)) // Only read one entry, using GetWith which should cache the root node _, err := GetWith(trie, keyValues[0].key, func(v []byte) []byte { return v }) assert.NoError(t, err) @@ -924,11 +916,10 @@ func Test_TrieDB(t *testing.T) { overlay := db.Clone() var newRoot hash.H256 { - trie := NewTrieDB(trie.rootHash, &overlay, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB(trie.rootHash, &overlay, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) for _, entry := range keyValues[1:] { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -941,14 +932,14 @@ func Test_TrieDB(t *testing.T) { for i, entry := range keyValues[1:] { cachedValue := cache.GetValue(entry.key) require.Equal(t, ExistingCachedValue[hash.H256]{ - Hash: runtime.BlakeTwo256{}.Hash(keyValues[i+1].value), + Hash: hasher.Blake2Hasher{}.Hash(keyValues[i+1].value), Data: keyValues[i+1].value, }, cachedValue) } partialDB := NewMemoryDB() for _, record := range recorder.Drain() { - // key := runtime.BlakeTwo256{}.Hash(record.Data).Bytes() + // key := hasher.Blake2Hasher{}.Hash(record.Data).Bytes() // require.NoError(t, partialDB.Set(key, record.Data)) partialDB.Insert(hashdb.EmptyPrefix, record.Data) } @@ -956,8 +947,7 @@ func Test_TrieDB(t *testing.T) { // Replay the it, but this time we use the proof. var validatedRoot hash.H256 { - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, partialDB) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, partialDB, version) for _, entry := range keyValues[1:] { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -972,8 +962,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("insert_remove_with_cache", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -991,11 +981,10 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) // Add all values for _, entry := range keyValues { @@ -1020,7 +1009,7 @@ func Test_TrieDB(t *testing.T) { require.NotNil(t, cachedValue) require.Equal(t, entry.value, cachedValue.data()) - require.Equal(t, runtime.BlakeTwo256{}.Hash(entry.value), *cachedValue.hash()) + require.Equal(t, hasher.Blake2Hasher{}.Hash(entry.value), *cachedValue.hash()) } for _, entry := range keyValues[3:] { @@ -1029,11 +1018,10 @@ func Test_TrieDB(t *testing.T) { // get values again using cache for _, entry := range keyValues[:3] { - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) require.NoError(t, err) require.NotNil(t, val) @@ -1044,8 +1032,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("insert_with_cache_more_nodes", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1065,11 +1053,10 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) // Add all values for _, entry := range keyValues { @@ -1089,16 +1076,15 @@ func Test_TrieDB(t *testing.T) { require.NotNil(t, cachedValue) require.Equal(t, entry.value, cachedValue.data()) - require.Equal(t, runtime.BlakeTwo256{}.Hash(entry.value), *cachedValue.hash()) + require.Equal(t, hasher.Blake2Hasher{}.Hash(entry.value), *cachedValue.hash()) } // get values again using cache for _, entry := range keyValues { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) require.NoError(t, err) require.NotNil(t, val) @@ -1109,8 +1095,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("insert_with_cache_insert_after_commit", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1130,11 +1116,10 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) // Add all values for _, entry := range keyValues { @@ -1154,16 +1139,15 @@ func Test_TrieDB(t *testing.T) { require.NotNil(t, cachedValue) require.Equal(t, entry.value, cachedValue.data()) - require.Equal(t, runtime.BlakeTwo256{}.Hash(entry.value), *cachedValue.hash()) + require.Equal(t, hasher.Blake2Hasher{}.Hash(entry.value), *cachedValue.hash()) } // get values again using cache for _, entry := range keyValues { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) require.NoError(t, err) require.NotNil(t, val) @@ -1173,11 +1157,10 @@ func Test_TrieDB(t *testing.T) { // ensure we can insert a new leaf node on a new triedb // use lookup functions to validate we're using cached version { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) require.NoError(t, trie.Set([]byte("AAB"), []byte{1, 1, 1, 1})) @@ -1202,8 +1185,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("insert_into_cache_and_lookup_using_cache", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1221,8 +1204,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) // Add all values for _, entry := range keyValues { @@ -1239,10 +1221,9 @@ func Test_TrieDB(t *testing.T) { // get all keys to populate cache { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trie.SetVersion(version) // get values again using cache for _, entry := range keyValues { val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) @@ -1258,16 +1239,15 @@ func Test_TrieDB(t *testing.T) { require.NotNil(t, cachedValue) require.Equal(t, entry.value, cachedValue.data()) - require.Equal(t, runtime.BlakeTwo256{}.Hash(entry.value), *cachedValue.hash()) + require.Equal(t, hasher.Blake2Hasher{}.Hash(entry.value), *cachedValue.hash()) } // get all keys again from cache, by passing in brand new db { db := NewMemoryDB() - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trie.SetVersion(version) // get values again using cache for _, entry := range keyValues { val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) @@ -1281,8 +1261,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("insert_into_cache_and_lookup_hash_using_cache", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1300,8 +1280,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) // Add all values for _, entry := range keyValues { @@ -1318,11 +1297,10 @@ func Test_TrieDB(t *testing.T) { // get all keys to populate cache { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trie.SetVersion(version) // get hashes for all entries populating cache for _, entry := range keyValues { h, err := trie.GetHash(entry.key) @@ -1334,10 +1312,9 @@ func Test_TrieDB(t *testing.T) { // get all keys again from cache, by passing in brand new db { db := NewMemoryDB() - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trie.SetVersion(version) // get hashes for all entries from cache for _, entry := range keyValues { h, err := trie.GetHash(entry.key) @@ -1348,10 +1325,9 @@ func Test_TrieDB(t *testing.T) { // get all values, by using cache and previous db { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trie.SetVersion(version) // get hashes for all entries from cache for _, entry := range keyValues { val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) @@ -1365,8 +1341,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("trie_nodes_recorded", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1381,8 +1357,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -1400,11 +1375,10 @@ func Test_TrieDB(t *testing.T) { recorder := NewRecorder[hash.H256]() { trie := NewTrieDB( - root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) for _, entry := range keyValues { if getHash { @@ -1435,9 +1409,9 @@ func Test_TrieDB(t *testing.T) { var isInline bool switch version { - case trie.V0: + case layoutV0{}: isInline = true - case trie.V1: + case layoutV1{}: if len(entry.value) > 32 { isInline = false } else { @@ -1461,8 +1435,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("trie_nodes_recorded_get_hashes_and_values", func(t *testing.T) { - for _, version := range []trie.TrieLayout{trie.V0, trie.V1} { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1477,8 +1451,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -1492,11 +1465,10 @@ func Test_TrieDB(t *testing.T) { recorder := NewRecorder[hash.H256]() { trie := NewTrieDB( - root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) for _, entry := range keyValues { h, err := trie.GetHash(entry.key) assert.NoError(t, err) @@ -1506,10 +1478,9 @@ func Test_TrieDB(t *testing.T) { // get all values, by using cache and previous db { - trie := NewTrieDB(root, db, - WithCache[hash.H256, runtime.BlakeTwo256](cache), + trie := NewTrieDB(root, db, version, + WithCache[hash.H256, hasher.Blake2Hasher](cache), ) - trie.SetVersion(version) // get values for all entries from cache for _, entry := range keyValues { val, err := GetWith(trie, entry.key, func(d []byte) []byte { return d }) @@ -1522,10 +1493,9 @@ func Test_TrieDB(t *testing.T) { // get all hashes, without using cache and previous db // pass in the recorder as well { - trie := NewTrieDB(root, db, - WithRecorder[hash.H256, runtime.BlakeTwo256](recorder), + trie := NewTrieDB(root, db, version, + WithRecorder[hash.H256, hasher.Blake2Hasher](recorder), ) - trie.SetVersion(version) for _, entry := range keyValues { h, err := trie.GetHash(entry.key) assert.NoError(t, err) @@ -1538,11 +1508,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("test_merkle_value_internal", func(t *testing.T) { - for _, version := range []trie.TrieLayout{ - trie.V0, - trie.V1, - } { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1559,8 +1526,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -1570,8 +1536,7 @@ func Test_TrieDB(t *testing.T) { root = trie.rootHash } - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) for _, entry := range keyValues { h, err := trie.LookupFirstDescendant(entry.key) require.NoError(t, err) @@ -1622,11 +1587,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("test_merkle_value_branches_internal", func(t *testing.T) { - for _, version := range []trie.TrieLayout{ - trie.V0, - trie.V1, - } { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1638,8 +1600,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -1649,8 +1610,7 @@ func Test_TrieDB(t *testing.T) { root = trie.rootHash } - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) // The hash is returned from the branch node. hash, err := trie.LookupFirstDescendant([]byte("A")) @@ -1670,11 +1630,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("test_merkle_value_empty_trie_internal", func(t *testing.T) { - for _, version := range []trie.TrieLayout{ - trie.V0, - trie.V1, - } { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1690,8 +1647,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) require.NoError(t, trie.Set(entry.key, entry.value)) @@ -1703,8 +1659,7 @@ func Test_TrieDB(t *testing.T) { } // Data set is empty. - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) hash, err := trie.LookupFirstDescendant([]byte("A")) require.NoError(t, err) require.Nil(t, hash) @@ -1730,11 +1685,8 @@ func Test_TrieDB(t *testing.T) { }) t.Run("test_merkle_value_modification_internal", func(t *testing.T) { - for _, version := range []trie.TrieLayout{ - trie.V0, - trie.V1, - } { - t.Run(version.String(), func(t *testing.T) { + for i, version := range []TrieLayout{layoutV0{}, layoutV1{}} { + t.Run(fmt.Sprintf("version%d", i), func(t *testing.T) { keyValues := []struct { key []byte value []byte @@ -1746,8 +1698,7 @@ func Test_TrieDB(t *testing.T) { db := NewMemoryDB() var root hash.H256 { - trie := NewEmptyTrieDB[hash.H256, runtime.BlakeTwo256](db) - trie.SetVersion(version) + trie := NewEmptyTrieDB[hash.H256, hasher.Blake2Hasher](db, version) for _, entry := range keyValues { require.NoError(t, trie.Set(entry.key, entry.value)) } @@ -1763,8 +1714,7 @@ func Test_TrieDB(t *testing.T) { aabaHashLHS MerkleValue[hash.H256] ) { - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) // The hash is returned from the branch node. hash, err := trie.LookupFirstDescendant([]byte("A")) @@ -1793,8 +1743,7 @@ func Test_TrieDB(t *testing.T) { ) // Modify AABA and expect AAAA to return the same merkle value { - trie := NewTrieDB[hash.H256, runtime.BlakeTwo256](root, db) - trie.SetVersion(version) + trie := NewTrieDB[hash.H256, hasher.Blake2Hasher](root, db, version) require.NoError(t, trie.Set([]byte("AABA"), bytes.Repeat([]byte{3}, 64))) err := trie.commit() require.NoError(t, err) diff --git a/pkg/trie/triedb/util_test.go b/pkg/trie/triedb/util_test.go index fe190dbb4f..a06d029207 100644 --- a/pkg/trie/triedb/util_test.go +++ b/pkg/trie/triedb/util_test.go @@ -6,14 +6,14 @@ package triedb import ( memorydb "github.com/ChainSafe/gossamer/internal/memory-db" chash "github.com/ChainSafe/gossamer/internal/primitives/core/hash" - "github.com/ChainSafe/gossamer/internal/primitives/runtime" + "github.com/ChainSafe/gossamer/internal/primitives/core/hasher" "github.com/ChainSafe/gossamer/pkg/trie/triedb/hash" ) func NewMemoryDB() *memorydb.MemoryDB[ - chash.H256, runtime.BlakeTwo256, chash.H256, memorydb.HashKey[chash.H256]] { + chash.H256, hasher.Blake2Hasher, chash.H256, memorydb.HashKey[chash.H256]] { db := memorydb.NewMemoryDB[ - chash.H256, runtime.BlakeTwo256, chash.H256, memorydb.HashKey[chash.H256], + chash.H256, hasher.Blake2Hasher, chash.H256, memorydb.HashKey[chash.H256], ]([]byte{0}) return &db }