diff --git a/Cargo.lock b/Cargo.lock index cae18e842..32f92997c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8065,6 +8065,8 @@ dependencies = [ "midnight-onchain-runtime 2.0.1", "midnight-onchain-runtime 3.1.0", "midnight-onchain-runtime 4.0.0", + "midnight-onchain-state 3.0.0", + "midnight-onchain-state 4.0.0", "midnight-primitives-ledger", "midnight-serialize", "midnight-storage 1.1.1", @@ -8114,6 +8116,8 @@ dependencies = [ "midnight-onchain-runtime 2.0.1", "midnight-onchain-runtime 3.1.0", "midnight-onchain-runtime 4.0.0", + "midnight-onchain-state 3.0.0", + "midnight-onchain-state 4.0.0", "midnight-serialize", "midnight-storage 1.1.1", "midnight-storage 2.0.1", @@ -8734,6 +8738,7 @@ checksum = "210e601a89aee79ce2b007cf96c1a56c23baa306b0c40b9b98d12c27e18d1981" dependencies = [ "crypto", "derive-where", + "hashbrown 0.16.1", "midnight-base-crypto", "midnight-serialize", "midnight-storage-core", @@ -8755,6 +8760,7 @@ dependencies = [ "crypto", "derive-where", "fake", + "hashbrown 0.16.1", "hex", "itertools 0.14.0", "konst 0.4.3", diff --git a/Cargo.toml b/Cargo.toml index 97892b688..35483639c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,8 +91,14 @@ zkir = { version = "^2.2.0", package = "midnight-zkir" } # Ledger 8 (compatible with layout-v2) # coin-structure and transient-crypto (2.x) share versions with L7 so reuse those entries. mn-ledger-8 = { version = "=8.1.0", package = "midnight-ledger" } -ledger-storage-ledger-8 = { version = "=2.0.1", package = "midnight-storage", features = ["parity-db"] } +# `state-translation` (needed by the v8->v9 storage migration) pulls in +# `public-internal-structure`, exposing `merkle_patricia_trie`/`storable`/the +# `state_translation` module used by `midnight_node_ledger::state_translation_v8_to_v9`. +ledger-storage-ledger-8 = { version = "=2.0.1", package = "midnight-storage", features = ["parity-db", "state-translation"] } onchain-runtime-ledger-8 = { version = "=3.1.0", package = "midnight-onchain-runtime" } +# Same `midnight-onchain-state` instance that `mn-ledger-8`'s `ContractState` +# resolves to (via onchain-runtime 3.1.0); used as the v8 side of the state translation table. +onchain-state-ledger-8 = { version = "=3.0.0", package = "midnight-onchain-state" } zswap-ledger-8 = { version = "=8.1.0", package = "midnight-zswap" } # Ledger 9 (compatible with layout-v2; midnight-ledger-v9 crate) @@ -101,6 +107,9 @@ zswap-ledger-8 = { version = "=8.1.0", package = "midnight-zswap" } # (midnight-zkir 2.2.0) serves all of L7/L8/L9. mn-ledger-9 = { version = "=1.0.0", package = "midnight-ledger-v9" } onchain-runtime-ledger-9 = { version = "=4.0.0", package = "midnight-onchain-runtime" } +# v9 side of the state translation table (matches `mn-ledger-9`'s `ContractState` +# via onchain-runtime 4.0.0). Patched to onchain-state-4.0.0-rc.3 by [patch.crates-io]. +onchain-state-ledger-9 = { version = "=4.0.0", package = "midnight-onchain-state" } zswap-ledger-9 = { version = "=9.0.0", package = "midnight-zswap" } coin-structure-ledger-9 = { version = "=3.0.0", package = "midnight-coin-structure" } transient-crypto-ledger-9 = { version = "=3.0.0", package = "midnight-transient-crypto" } diff --git a/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md new file mode 100644 index 000000000..e2c92f72c --- /dev/null +++ b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md @@ -0,0 +1,22 @@ +#node #runtime #ledger + +# On-chain ledger 8->9 hardfork state migration + +Lets a ledger-8 chain (e.g. `1.0.1`) runtime-upgrade in place to the current +ledger-9 runtime. A new host fn, `migrate_state_v8_to_v9`, runs the +`StateTranslationTable` (ported from `midnight-ledger` PR #539) to translate +the on-chain `LedgerState` from v13 to v18. It's wired in as +`pallet_midnight::migrations::v2::MigrateV1ToV2`, a `VersionedMigration<1,2,..>` +that fires once when a ledger-8 chain (pallet-midnight storage version 1) +upgrades to this runtime (storage version 2); a fresh ledger-9 genesis starts +at version 2 and skips it. The migration's weight is derived from the ledger +cost model rather than a hand-tuned estimate. + +Also includes two fixes needed to support both ledger-8 and ledger-9 chains: +version-aware genesis seeding (detected via `serialize::peek_tag` instead of +hardcoding the v9 deserializer), and restoring the ledger-8 +`construct_distribute_treasury_system_tx` host fn, which the `1.0.1` WASM +still imports. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1925 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1580 diff --git a/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md b/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md new file mode 100644 index 000000000..9fc711c0e --- /dev/null +++ b/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md @@ -0,0 +1,18 @@ +#toolkit #ledger9 + +# Fork-aware transaction generation across the ledger 8->9 hardfork + +`replay_blocks` now detects the v8->v9 fork boundary and runs the same +`StateTranslationTable` translation used by the on-chain migration +(`fork_context_8_to_9` / `fork_8_to_9_if_needed`), so transactions generated +after the fork are built against the correctly-translated ledger-9 context +instead of a stale ledger-8 one. + +The `runtime-upgrade` command now waits for the new runtime to actually +*execute* at a finalized block, not just be applied/stored. The stored spec +version flips at the apply block, but that block still executes under the +old runtime, so polling only the stored spec left transaction generation +reading a block short of any ledger-9-classified block. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1925 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1580 diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 4cbea9a5f..db58c54d7 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -18,11 +18,13 @@ zswap = { workspace = true, optional = true } mn-ledger-8 = { workspace = true, features = ["proving"], optional = true } onchain-runtime-ledger-8 = { workspace = true, optional = true } +onchain-state-ledger-8 = { workspace = true, optional = true } ledger-storage-ledger-8 = { workspace = true, optional = true } zswap-ledger-8 = { workspace = true, optional = true } mn-ledger-9 = { workspace = true, features = ["proving"], optional = true } onchain-runtime-ledger-9 = { workspace = true, optional = true } +onchain-state-ledger-9 = { workspace = true, optional = true } zswap-ledger-9 = { workspace = true, optional = true } coin-structure-ledger-9 = { workspace = true, optional = true } transient-crypto-ledger-9 = { workspace = true, optional = true } @@ -49,6 +51,11 @@ scale-info.workspace = true [dev-dependencies] midnight-node-res = { workspace = true, features = ["test", "chain-spec"] } +# The crate's own `#[cfg(test)]` modules (api::ledger, api::transaction, and the +# state-translation tests) use `extract_tx_with_context`, gated behind the +# helpers `can-panic` feature. Enable it for test builds so the tests compile +# when the crate is tested standalone. +midnight-node-ledger-helpers = { workspace = true, features = ["can-panic", "test-utils"] } [features] default = [ @@ -81,10 +88,12 @@ std = [ "zswap", "mn-ledger-8", "onchain-runtime-ledger-8", + "onchain-state-ledger-8", "ledger-storage-ledger-8", "zswap-ledger-8", "mn-ledger-9", "onchain-runtime-ledger-9", + "onchain-state-ledger-9", "zswap-ledger-9", "coin-structure-ledger-9", "transient-crypto-ledger-9", diff --git a/ledger/helpers/Cargo.toml b/ledger/helpers/Cargo.toml index 76c286887..4d38158c9 100644 --- a/ledger/helpers/Cargo.toml +++ b/ledger/helpers/Cargo.toml @@ -27,11 +27,13 @@ reqwest = { workspace = true } mn-ledger-8 = { workspace = true, features = ["proving"] } onchain-runtime-ledger-8 = { workspace = true } +onchain-state-ledger-8 = { workspace = true } ledger-storage-ledger-8 = { workspace = true } zswap-ledger-8 = { workspace = true } mn-ledger-9 = { workspace = true, features = ["proving", "test-utilities"] } onchain-runtime-ledger-9 = { workspace = true } +onchain-state-ledger-9 = { workspace = true } zswap-ledger-9 = { workspace = true } coin-structure-ledger-9 = { workspace = true } transient-crypto-ledger-9 = { workspace = true } diff --git a/ledger/helpers/src/fork/fork_8_to_9.rs b/ledger/helpers/src/fork/fork_8_to_9.rs index 918cda2e9..fac03ec49 100644 --- a/ledger/helpers/src/fork/fork_8_to_9.rs +++ b/ledger/helpers/src/fork/fork_8_to_9.rs @@ -2,6 +2,10 @@ use std::collections::HashMap; use tokio::sync::Mutex as MutexTokio; +use crate::state_translation_v8_to_v9::StateTranslationTable; +use base_crypto::cost_model::CostDuration; +use ledger_storage_ledger_8::state_translation::TypedTranslationState; + type Db8 = crate::ledger_8::DefaultDB; type Db9 = crate::ledger_9::DefaultDB; @@ -67,8 +71,35 @@ pub fn fork_context_8_to_9( context8: LedgerContext8, ) -> Result, std::io::Error> { let ledger_state_8 = context8.ledger_state.lock().expect("failed to lock ledger state"); - let ledger_state: crate::ledger_9::Sp, Db8> = - old_to_new_sp(ledger_state_8.clone())?; + // Real v8->v9 state translation (NOT `old_to_new_sp`): the LedgerState tag + // changed v13->v18 and its shape changed, so a bare arena-key reuse would + // produce a v9 root the ledger-9 machinery can't read. Walk the v8 state + // through the same `StateTranslationTable` the on-chain migration uses. Db8 + // == Db9 (both `ledger_storage_ledger_8::DefaultDB`), so source and target + // share one arena and a single-`D` `TypedTranslationState` applies. + let ledger_state: crate::ledger_9::Sp, Db8> = { + let mut tl = TypedTranslationState::< + mn_ledger_8::structure::LedgerState, + mn_ledger_9::structure::LedgerState, + StateTranslationTable, + Db8, + >::start(ledger_state_8.clone())?; + // Single-shot: a generous per-step budget drains the whole state in a + // couple of iterations; the step cap is only a runaway backstop. + // 1_000_000_000_000 pico-seconds == 1 second + let budget = CostDuration::from_picoseconds(1_000_000_000_000); + let mut steps = 0usize; + loop { + steps += 1; + if steps > 100_000 { + return Err(std::io::Error::other("v8->v9 state translation did not converge")); + } + tl = tl.run(budget)?; + if let Some(result) = tl.result()? { + break result; + } + } + }; let mut wallets = HashMap::new(); for (k, v) in context8.wallets.lock().expect("failed to lock wallets").iter() { diff --git a/ledger/helpers/src/lib.rs b/ledger/helpers/src/lib.rs index 9f232d4ea..bb8a18dc9 100644 --- a/ledger/helpers/src/lib.rs +++ b/ledger/helpers/src/lib.rs @@ -16,6 +16,11 @@ mod utils; pub use utils::find_dependency_version; pub mod extract_tx_with_context; +/// v8 -> v9 ledger state translation table (ported from midnight-ledger PR #539). +/// Consumed by the runtime storage migration (via the `ledger` crate) and by the +/// toolkit fork boundary (`fork::fork_8_to_9`). +pub mod state_translation_v8_to_v9; + /// Strategy for ordering candidate coins/UTXOs during input selection. /// /// Defined at the crate root (not inside the version-specific `common` module) so that diff --git a/ledger/helpers/src/state_translation_v8_to_v9.rs b/ledger/helpers/src/state_translation_v8_to_v9.rs new file mode 100644 index 000000000..4106b7819 --- /dev/null +++ b/ledger/helpers/src/state_translation_v8_to_v9.rs @@ -0,0 +1,710 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! State translation from ledger v8 to ledger v9. +//! +//! Ported from `midnight-ledger` PR #539 (`v8-to-v9-state-translation`). The +//! only changes from the upstream crate are the import aliases below, which map +//! the translation's `ledger_v8` / `ledger_v9` / `onchain_state_v8` / +//! `onchain_state_v9` / `storage` / `serialize` crate names onto this +//! workspace's package aliases. The [`StateTranslationTable`] is consumed by the +//! v8->v9 storage migration ([`crate::host_api::migration_8_to_9`]). +//! +//! ## State shape differences (only stored types listed) +//! +//! | type | v8 tag | v9 tag | change | +//! | ---------------------------- | ------------------------------------ | ------------------------------------ | ------ | +//! | LedgerState | `ledger-state[v13]` | `ledger-state[v18]` | `bridge_receiving` map gains `NightAnn` | +//! | LedgerParameters | `ledger-parameters[v5]` | `ledger-parameters[v8]` | adds `min_block_price`; `TransactionLimits` adds `max_contract_metadata_size`; `TransactionCostModel` drops `parallelism_factor`, adds `validation`/`guaranteed`/`fallible` factors | +//! | ContractState | `contract-state[v6]` | `contract-state[v8]` | reflows `ContractOperation` + `ContractMaintenanceAuthority` changes | +//! | ContractOperation | `contract-operation[v4]` | `contract-operation[v6]` | single `v2` key -> `{ v2, v3, ir }`; v8 key maps to `v2`, new `v3`/`ir` empty | +//! | ContractMaintenanceAuthority | `contract-maintenance-authority[v1]` | `contract-maintenance-authority[v2]` | `committee: Vec` -> `Vec` (Schnorr/ECDSA sum) | +//! +//! Everything else (zswap, utxo, dust, replay_protection, treasury, +//! unclaimed_block_rewards) is tag-stable and passes through `recast`. + +// Map the upstream translation crate names onto the node workspace's package +// aliases. `mn-ledger-8`/`mn-ledger-9` are the two `midnight-ledger` majors; +// `onchain-state-ledger-8`/`-9` are the exact `midnight-onchain-state` instances +// that each ledger's `ContractState` resolves to; `ledger-storage-ledger-8` +// (midnight-storage 2.0.1, `state-translation` feature) backs both. +use ledger_storage_ledger_8 as storage; +use midnight_serialize as serialize; +use mn_ledger_8 as ledger_v8; +use mn_ledger_9 as ledger_v9; +use onchain_state_ledger_8 as onchain_state_v8; +use onchain_state_ledger_9 as onchain_state_v9; + +use base_crypto::cost_model::CostDuration; +use serialize::Tagged; +use std::ops::Deref; +use std::{any::Any, borrow::Cow, io, marker::PhantomData}; +use storage::{ + Storable, + arena::Sp, + db::DB, + merkle_patricia_trie::{self, Annotation, MerklePatriciaTrie}, + state_translation::*, + storable::SizeAnn, + storage::{HashMap, Map, default_storage}, +}; + +// ---------- Generic helpers (copied from the v6->v7 reference) ---------- + +/// Recast a stored object from one type to another, requiring matching tags. +/// Used for subtrees whose tag is unchanged between v8 and v9. +fn recast + Tagged, B: Storable + Tagged, D: DB>( + a: &Sp, +) -> io::Result> { + if A::tag() != B::tag() { + return io::Result::Err(io::Error::other("tags do not match")); + } + default_storage::().get_lazy(&a.as_child().into()) +} + +/// Generic MPT translation: walks the trie, translating each entry via the +/// table-registered translation for `A->B`, and recomputes annotations under +/// `AnnB` from the new values. +struct MptTl(PhantomData<(A, B, AnnA, AnnB)>); + +impl< + A: Storable + Tagged, + B: Storable + Tagged, + AnnA: Annotation + Storable + Tagged, + AnnB: Annotation + Storable + Tagged, + D: DB, +> DirectTranslation, MerklePatriciaTrie, D> + for MptTl +{ + fn required_translations() -> Vec { + vec![TranslationId( + merkle_patricia_trie::Node::::tag(), + merkle_patricia_trie::Node::::tag(), + )] + } + fn child_translations( + source: &MerklePatriciaTrie, + ) -> Vec<(TranslationId, Sp)> { + let tlids = , _, D>>::required_translations(); + vec![(tlids[0].clone(), source.0.upcast())] + } + fn finalize( + source: &MerklePatriciaTrie, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let tls = Self::child_translations(source); + Ok(Some(MerklePatriciaTrie(try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child()))))) + } +} + +impl< + A: Storable + Tagged, + B: Storable + Tagged, + AnnA: Storable + Tagged + Annotation, + AnnB: Storable + Tagged + Annotation, + D: DB, +> + DirectTranslation< + merkle_patricia_trie::Node, + merkle_patricia_trie::Node, + D, + > for MptTl +{ + fn required_translations() -> Vec { + let entry_tl = TranslationId(A::tag(), B::tag()); + let self_tl = TranslationId( + merkle_patricia_trie::Node::::tag(), + merkle_patricia_trie::Node::::tag(), + ); + vec![entry_tl, self_tl] + } + fn child_translations( + source: &merkle_patricia_trie::Node, + ) -> Vec<(TranslationId, Sp)> { + let tls = , _, D>>::required_translations(); + let entry_tl = tls[0].clone(); + let self_tl = tls[1].clone(); + match source { + merkle_patricia_trie::Node::Empty => vec![], + merkle_patricia_trie::Node::Branch { children, .. } => { + children.iter().map(|child| (self_tl.clone(), child.upcast())).collect() + }, + merkle_patricia_trie::Node::Extension { child, .. } => { + vec![(self_tl, child.upcast())] + }, + merkle_patricia_trie::Node::MidBranchLeaf { value, child, .. } => { + vec![(entry_tl, value.upcast()), (self_tl, child.upcast())] + }, + merkle_patricia_trie::Node::Leaf { value, .. } => vec![(entry_tl, value.upcast())], + } + } + fn finalize( + source: &merkle_patricia_trie::Node, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let tls = Self::child_translations(source); + Ok(Some(match source { + merkle_patricia_trie::Node::Empty => merkle_patricia_trie::Node::Empty, + merkle_patricia_trie::Node::Branch { .. } => { + let mut new_children = + core::array::from_fn(|_| Sp::new(merkle_patricia_trie::Node::Empty)); + for (child, new_child) in tls.iter().zip(new_children.iter_mut()) { + *new_child = try_resopt!(cache.resolve(&child.0, child.1.as_child())); + } + let ann = new_children.iter().fold(AnnB::empty(), |acc, x| { + acc.append(&merkle_patricia_trie::Node::::ann(x)) + }); + merkle_patricia_trie::Node::Branch { ann, children: Box::new(new_children) } + }, + merkle_patricia_trie::Node::Extension { compressed_path, .. } => { + let child: Sp, D> = + try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let ann = merkle_patricia_trie::Node::::ann(&child); + merkle_patricia_trie::Node::Extension { + ann, + compressed_path: compressed_path.clone(), + child, + } + }, + merkle_patricia_trie::Node::Leaf { .. } => { + let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let ann = AnnB::from_value(&value); + merkle_patricia_trie::Node::Leaf { ann, value } + }, + merkle_patricia_trie::Node::MidBranchLeaf { .. } => { + let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let child: Sp, D> = + try_resopt!(cache.resolve(&tls[1].0, tls[1].1.as_child())); + let ann = AnnB::from_value(&value) + .append(&merkle_patricia_trie::Node::::ann(&child)); + merkle_patricia_trie::Node::MidBranchLeaf { ann, value, child } + }, + })) + } +} + +/// Identity translation for a type whose serialization is unchanged across +/// versions. Needed when an MPT's entries are tag-stable but its annotation +/// changes (e.g. `bridge_receiving`). +struct IdentityTl(PhantomData); + +impl + Clone, D: DB> DirectTranslation for IdentityTl { + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations(_: &T) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &T, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result> { + Ok(Some(source.clone())) + } +} + +// ---------- Translation IDs (shorthand) ---------- + +struct Ids; + +impl Ids { + fn contract_mpt() -> TranslationId { + TranslationId( + MerklePatriciaTrie::< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >::tag(), + MerklePatriciaTrie::< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >::tag(), + ) + } + + fn bridge_receiving_mpt() -> TranslationId { + TranslationId( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ) + } + + fn parameters() -> TranslationId { + TranslationId( + ledger_v8::structure::LedgerParameters::tag(), + ledger_v9::structure::LedgerParameters::tag(), + ) + } +} + +// ---------- Top-level: LedgerState v8 -> v9 ---------- + +struct LedgerStateTl; + +impl + DirectTranslation, ledger_v9::structure::LedgerState, D> + for LedgerStateTl +{ + fn required_translations() -> Vec { + vec![Ids::parameters(), Ids::bridge_receiving_mpt::(), Ids::contract_mpt::()] + } + + fn child_translations( + source: &ledger_v8::structure::LedgerState, + ) -> Vec<(TranslationId, Sp)> { + vec![ + (Ids::parameters(), source.parameters.upcast()), + (Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.upcast()), + (Ids::contract_mpt::(), source.contract.mpt.upcast()), + ] + } + + fn finalize( + source: &ledger_v8::structure::LedgerState, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let Some(parameters) = cache.lookup(&Ids::parameters(), source.parameters.as_child()) + else { + return Ok(None); + }; + let Some(bridge_recv_mpt) = + cache.lookup(&Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.as_child()) + else { + return Ok(None); + }; + let Some(contract_mpt) = + cache.lookup(&Ids::contract_mpt::(), source.contract.mpt.as_child()) + else { + return Ok(None); + }; + + Ok(Some(ledger_v9::structure::LedgerState { + network_id: source.network_id.clone(), + parameters: parameters.force_downcast(), + locked_pool: source.locked_pool, + bridge_receiving: Map { mpt: bridge_recv_mpt.force_downcast(), key_type: PhantomData }, + reserve_pool: source.reserve_pool, + block_reward_pool: source.block_reward_pool, + unclaimed_block_rewards: Map { + mpt: recast(&source.unclaimed_block_rewards.mpt)?, + key_type: PhantomData, + }, + treasury: Map { mpt: recast(&source.treasury.mpt)?, key_type: PhantomData }, + zswap: recast(&source.zswap)?, + contract: Map { mpt: contract_mpt.force_downcast(), key_type: PhantomData }, + utxo: recast(&source.utxo)?, + replay_protection: recast(&source.replay_protection)?, + dust: recast(&source.dust)?, + })) + } +} + +// ---------- LedgerParameters v8 -> v9 ---------- + +struct LedgerParametersTl; + +impl + DirectTranslation< + ledger_v8::structure::LedgerParameters, + ledger_v9::structure::LedgerParameters, + D, + > for LedgerParametersTl +{ + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations( + _: &ledger_v8::structure::LedgerParameters, + ) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &ledger_v8::structure::LedgerParameters, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result> { + // Base-crypto-backed fields (Duration, FixedPoint, primitives) are + // assignable directly because `midnight-base-crypto` is unified across + // v8 and v9 by workspace patches. Composite types defined in `ledger` + // (TransactionCostModel, dust parameters, etc.) are tag-stable but not + // identical types, so we go through the (de)serializer. + // + // `TransactionLimits` is the exception: v9 bumped it to + // `transaction-limits[v3]` by adding `max_contract_metadata_size`, so + // it is no longer tag-stable and is rebuilt field-by-field (its other + // fields are unified base-crypto types). + Ok(Some(ledger_v9::structure::LedgerParameters { + // `TransactionCostModel` bumped `transaction-cost-model[v4]`->`[v5]`: + // v9 drops `parallelism_factor` and adds three `FixedPoint` factors. + // The two surviving fields are tag-stable and recast through; the new + // factors get the v9 INITIAL_PARAMETERS defaults. + cost_model: ledger_v9::structure::TransactionCostModel { + runtime_cost_model: recast_base(&source.cost_model.runtime_cost_model)?, + baseline_cost: recast_base(&source.cost_model.baseline_cost)?, + // NEW IN v9 — placeholder; the production value should match the + // value chosen for the hardfork. + validation_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .validation_factor, + guaranteed_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .guaranteed_factor, + fallible_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .fallible_factor, + }, + limits: ledger_v9::structure::TransactionLimits { + transaction_byte_limit: source.limits.transaction_byte_limit, + time_to_dismiss_per_byte: source.limits.time_to_dismiss_per_byte, + min_time_to_dismiss: source.limits.min_time_to_dismiss, + block_limits: source.limits.block_limits, + block_withdrawal_minimum_multiple: source.limits.block_withdrawal_minimum_multiple, + // NEW IN v9 — placeholder; the production value should match + // the value chosen for the hardfork. + max_contract_metadata_size: ledger_v9::structure::INITIAL_PARAMETERS + .limits + .max_contract_metadata_size, + }, + dust: recast_base(&source.dust)?, + fee_prices: recast_base(&source.fee_prices)?, + global_ttl: source.global_ttl, + cost_dimension_min_ratio: source.cost_dimension_min_ratio, + price_adjustment_a_parameter: source.price_adjustment_a_parameter, + cardano_to_midnight_bridge_fee_basis_points: source + .cardano_to_midnight_bridge_fee_basis_points, + c_to_m_bridge_min_amount: source.c_to_m_bridge_min_amount, + // NEW IN v9 — placeholder; the production value should match the + // value chosen for the hardfork. + min_block_price: ledger_v9::structure::INITIAL_PARAMETERS.min_block_price, + })) + } +} + +/// Recast for tag-stable base types passed by value (cost model, limits, etc.). +/// Not the same as `recast` above which only works for `Sp`. +fn recast_base( + a: &A, +) -> io::Result { + if A::tag() != B::tag() { + return Err(io::Error::other("tags do not match")); + } + let mut buf = Vec::new(); + a.serialize(&mut buf)?; + B::deserialize(&mut &buf[..], 0) +} + +// ---------- ContractOperation v8 -> v9 ---------- + +/// Translate a single contract operation. v9 grew `ContractOperation` from a +/// single `v2` verifier key (`contract-operation[v4]`) to `{ v2, v3, ir }` +/// (`contract-operation[v6]`). v8's only key is a zk-stdlib-v1 key +/// (`verifier-key[v6]`), which v9 keeps in its `v2` slot: that slot is backed by +/// the same `transient-crypto` 2.x crate (`transient_crypto_old`), so it is the +/// identical type and assigns directly. The new zk-stdlib-v2 `v3` key +/// (`verifier-key[v7]`, transient-crypto 3.x) and the `ir` slot have no v8 +/// equivalent and stay empty — v9 keys are *not* synthesized from v8 keys. +/// (Note `ContractOperation::new(vk, ir)` sets `v3`, not `v2`, so the struct is +/// built field-wise here.) +fn translate_contract_operation( + source: &onchain_state_v8::state::ContractOperation, +) -> onchain_state_v9::state::ContractOperation { + // `ContractOperation` is `#[non_exhaustive]`; `new` seeds `v3`/`ir`, and the + // v8 key goes into the `v2` slot field-wise. + let mut op = onchain_state_v9::state::ContractOperation::new(None, None); + op.v2 = source.v2.clone(); + op +} + +// ---------- ContractState v8 -> v9 ---------- + +struct ContractStateTl; + +impl + DirectTranslation< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + D, + > for ContractStateTl +{ + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations( + _: &onchain_state_v8::state::ContractState, + ) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &onchain_state_v8::state::ContractState, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result>> { + // `operations` entries (ContractOperation) changed shape, so the map + // is rebuilt entry-by-entry. The translation machinery can't walk these + // base-storable leaves nested under a contract, but a contract's + // operation set is small, so an in-place rebuild is fine. ChargedState + // and the balance map (keyed u128) are tag-stable and recast through. + let mut operations = HashMap::new(); + for entry in source.operations.iter() { + let (key, op) = &*entry; + let key_v9: onchain_state_v9::state::EntryPointBuf = key[..].into(); + operations = operations.insert(key_v9, translate_contract_operation(op)); + } + let committee_v9 = source + .maintenance_authority + .committee + .iter() + .map(|vk| onchain_state_v9::state::ContractMaintenanceVerifyingKey::Schnorr(vk.clone())) + .collect(); + let maintenance_authority = onchain_state_v9::state::ContractMaintenanceAuthority { + committee: committee_v9, + threshold: source.maintenance_authority.threshold, + counter: source.maintenance_authority.counter, + }; + Ok(Some(onchain_state_v9::state::ContractState:: { + data: recast::< + onchain_state_v8::state::ChargedState, + onchain_state_v9::state::ChargedState, + D, + >(&Sp::new(source.data.clone()))? + .deref() + .clone(), + operations, + maintenance_authority, + balance: HashMap(Map { mpt: recast(&source.balance.0.mpt)?, key_type: PhantomData }), + })) + } +} + +// ---------- Translation table ---------- + +pub struct StateTranslationTable; + +impl TranslationTable for StateTranslationTable { + const TABLE: &[(TranslationId, &dyn TypelessTranslation)] = &[ + // Top-level + ( + TranslationId(Cow::Borrowed("ledger-state[v13]"), Cow::Borrowed("ledger-state[v18]")), + &DirectSpTranslation::<_, _, LedgerStateTl, _>(PhantomData), + ), + // LedgerParameters + ( + TranslationId( + Cow::Borrowed("ledger-parameters[v5]"), + Cow::Borrowed("ledger-parameters[v8]"), + ), + &DirectSpTranslation::<_, _, LedgerParametersTl, _>(PhantomData), + ), + // ContractState + ( + TranslationId(Cow::Borrowed("contract-state[v6]"), Cow::Borrowed("contract-state[v8]")), + &DirectSpTranslation::<_, _, ContractStateTl, _>(PhantomData), + ), + // `contract` MPT in LedgerState — entries are ContractState + ( + TranslationId( + Cow::Borrowed("mpt(contract-state[v6],night-annotation)"), + Cow::Borrowed("mpt(contract-state[v8],night-annotation)"), + ), + &DirectSpTranslation::< + MerklePatriciaTrie< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >, + MerklePatriciaTrie< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >, + MptTl< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + ledger_v8::annotation::NightAnn, + ledger_v9::annotation::NightAnn, + >, + _, + >(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt-node(contract-state[v6],night-annotation)"), + Cow::Borrowed("mpt-node(contract-state[v8],night-annotation)"), + ), + &DirectSpTranslation::< + merkle_patricia_trie::Node< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >, + merkle_patricia_trie::Node< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >, + MptTl< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + ledger_v8::annotation::NightAnn, + ledger_v9::annotation::NightAnn, + >, + _, + >(PhantomData), + ), + // `bridge_receiving` MPT — entries unchanged (u128), annotation changes + // from SizeAnn to NightAnn. Needs an identity entry translation and an + // MptTl that re-annotates. + ( + TranslationId(Cow::Borrowed("u128"), Cow::Borrowed("u128")), + &DirectSpTranslation::, _>(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt(u128,size-annotation)"), + Cow::Borrowed("mpt(u128,night-annotation)"), + ), + &DirectSpTranslation::< + MerklePatriciaTrie, + MerklePatriciaTrie, + MptTl, + _, + >(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt-node(u128,size-annotation)"), + Cow::Borrowed("mpt-node(u128,night-annotation)"), + ), + &DirectSpTranslation::< + merkle_patricia_trie::Node, + merkle_patricia_trie::Node, + MptTl, + _, + >(PhantomData), + ), + ]; +} + +#[cfg(test)] +mod tests { + use super::*; + use storage::db::InMemoryDB; + + fn translate_to_completion( + v8: ledger_v8::structure::LedgerState, + ) -> ledger_v9::structure::LedgerState { + let tl_state = TypedTranslationState::< + ledger_v8::structure::LedgerState, + ledger_v9::structure::LedgerState, + StateTranslationTable, + InMemoryDB, + >::start(Sp::new(v8)) + .expect("Failed to start translation"); + + let cost = CostDuration::from_picoseconds(1_000_000_000_000); + let finished = tl_state.run(cost).expect("Translation failed"); + + finished + .result() + .expect("Failed to get result") + .expect("Translation did not complete") + .deref() + .clone() + } + + /// Every `TranslationId` a table entry requires must itself be in the table, + /// or translation errors at runtime the first time the entry is needed. + #[test] + fn table_is_closed() { + >::assert_closure(); + } + + /// The `TABLE` hardcodes tag string literals. If a tag on either the v8 or v9 + /// side drifts (e.g. an rc bump changes a `#[tag]`), the literal no longer + /// matches what `T::tag()` produces and the migration silently mis-dispatches. + /// Rebuild every expected ID from the node's actual crate types and compare. + #[test] + fn table_tags_match_types() { + use storage::merkle_patricia_trie::{MerklePatriciaTrie, Node}; + use storage::storable::SizeAnn; + + type V8Ann = ledger_v8::annotation::NightAnn; + type V9Ann = ledger_v9::annotation::NightAnn; + type V8Contract = onchain_state_v8::state::ContractState; + type V9Contract = onchain_state_v9::state::ContractState; + + let expected: Vec<(Cow<'static, str>, Cow<'static, str>)> = vec![ + ( + ledger_v8::structure::LedgerState::::tag(), + ledger_v9::structure::LedgerState::::tag(), + ), + ( + ledger_v8::structure::LedgerParameters::tag(), + ledger_v9::structure::LedgerParameters::tag(), + ), + (V8Contract::tag(), V9Contract::tag()), + ( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ), + ( + Node::::tag(), + Node::::tag(), + ), + (u128::tag(), u128::tag()), + ( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ), + (Node::::tag(), Node::::tag()), + ]; + + let actual: Vec<_> = >::TABLE + .iter() + .map(|(id, _)| (id.0.clone(), id.1.clone())) + .collect(); + + assert_eq!(actual, expected); + } + + /// End-to-end smoke test: a default v8 `LedgerState` translates to v9, + /// preserving the tag-stable pools and picking up the new v9 default + /// `min_block_price`, and survives a v9 serialize round-trip. + #[test] + fn empty_state_translates_and_round_trips() { + let v8 = ledger_v8::structure::LedgerState::::new("test-network"); + let v9 = translate_to_completion(v8.clone()); + + assert_eq!(v9.network_id, v8.network_id); + assert_eq!(v9.reserve_pool, v8.reserve_pool); + assert_eq!(v9.locked_pool, v8.locked_pool); + assert_eq!(v9.block_reward_pool, v8.block_reward_pool); + assert_eq!( + v9.parameters.min_block_price, + ledger_v9::structure::INITIAL_PARAMETERS.min_block_price, + ); + + let mut buf = Vec::new(); + serialize::tagged_serialize(&v9, &mut buf).expect("v9 serialize"); + let v9_rt: ledger_v9::structure::LedgerState = + serialize::tagged_deserialize(&mut &buf[..]).expect("v9 deserialize"); + assert_eq!(v9_rt.network_id, v9.network_id); + } +} diff --git a/ledger/src/host_api/ledger_8.rs b/ledger/src/host_api/ledger_8.rs index 13bf1a9c0..0e05b0d20 100644 --- a/ledger/src/host_api/ledger_8.rs +++ b/ledger/src/host_api/ledger_8.rs @@ -461,6 +461,20 @@ pub trait Ledger8Bridge { } } + /// The ledger-8 runtime imports this to pay block rewards to the treasury. + /// Retained (removed for v9) so the current node can execute the ledger-8 + /// WASM across the 8->9 hardfork boundary. + fn construct_distribute_treasury_system_tx( + &mut self, + amount: PassFatPointerAndDecode, + ) -> AllocateAndReturnByCodec, LedgerApiError>> { + if is_unified(*self) { + Bridge::::construct_distribute_treasury_system_tx(amount) + } else { + Bridge::::construct_distribute_treasury_system_tx(amount) + } + } + /// Ensures the correct ledger storage is initialized for this runtime version. /// Handles rollback: if new version's storage is initialized but we need this version's storage, /// drops new version's storage and initializes normal storage. diff --git a/ledger/src/host_api/ledger_9.rs b/ledger/src/host_api/ledger_9.rs index 35bf5617b..669c675ef 100644 --- a/ledger/src/host_api/ledger_9.rs +++ b/ledger/src/host_api/ledger_9.rs @@ -501,6 +501,32 @@ pub trait Ledger9Bridge { true } + /// Translate the ledger state from ledger-v8 format to ledger-v9 format. + /// + /// Called by `pallet_midnight`'s v8->v9 storage migration during the runtime + /// upgrade that crosses into ledger-9. `state_key` is the pallet's `StateKey` + /// (a v8 arena root); returns the new v9 arena root to store back, together + /// with the synthetic cost (picoseconds) the translation consumed against + /// the ledger's cost model, for the pallet to charge as this migration's + /// weight. + fn migrate_state_v8_to_v9( + &mut self, + state_key: PassFatPointerAndRead<&[u8]>, + ) -> AllocateAndReturnByCodec, u64), LedgerApiError>> { + // Ensure the ledger arena is initialized before translating. The migration + // runs in the Executive migrations tuple, before pallet_midnight's + // on_initialize/on_runtime_upgrade have (re)initialized storage this block. + // `set_default_storage` is idempotent — a no-op if the pre-fork ledger-8 + // blocks already set it (v8 and v9 share the same storage backend). + if is_unified(*self) { + Bridge::::set_default_storage(*self); + crate::host_api::migration_8_to_9::migrate_state_v8_to_v9::(state_key) + } else { + Bridge::::set_default_storage(*self); + crate::host_api::migration_8_to_9::migrate_state_v8_to_v9::(state_key) + } + } + /// Initialize a process-wide temporary ledger ParityDb seeded with the /// undeployed-network genesis state. /// diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs new file mode 100644 index 000000000..6fe05c873 --- /dev/null +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -0,0 +1,222 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Host-side v8 -> v9 ledger state translation, driven by +//! [`crate::state_translation_v8_to_v9::StateTranslationTable`]. +//! +//! The on-chain pallet stores only the arena root of the ledger state +//! (`pallet_midnight::StateKey`, a `tagged_serialize`d `TypedArenaKey`); +//! the `LedgerState` itself lives in the process-global ledger arena (parity-db). +//! When the runtime upgrades from a ledger-8 runtime (spec < 2_000_000) to a +//! ledger-9 runtime (spec >= 2_000_000), the on-chain migration +//! (`pallet_midnight::migrations`) calls [`Ledger9Bridge::migrate_state_v8_to_v9`] +//! which lands here: it reads the v8 arena root, walks the v8 `LedgerState` +//! translating it into a v9 `LedgerState`, re-persists it, and returns the new v9 +//! arena root for the pallet to store back into `StateKey`. +//! +//! v8 and v9 share one storage crate (`ledger-storage-ledger-8`, +//! midnight-storage 2.0.1) and hence one arena, so the translation reads and +//! writes the same parity-db instance the pre-fork ledger-8 blocks populated. + +use crate::ledger_8::api::Ledger as Ledger8; +use crate::ledger_9::api::Ledger as Ledger9; +use crate::ledger_9::types::{DeserializationError, LedgerApiError, SerializationError}; +use midnight_node_ledger_helpers::state_translation_v8_to_v9::StateTranslationTable; + +use base_crypto::cost_model::CostDuration; +use ledger_storage_ledger_8 as storage; +use midnight_serialize::{tagged_deserialize, tagged_serialize}; +use storage::{ + arena::{Sp, TypedArenaKey}, + db::DB, + state_translation::TypedTranslationState, + storage::default_storage, +}; + +type LedgerState8 = mn_ledger_8::structure::LedgerState; +type LedgerState9 = mn_ledger_9::structure::LedgerState; + +const LOG_TARGET: &str = "midnight::ledger::migration_8_to_9"; + +/// Picoseconds granted to each `TypedTranslationState::run` step. The migration +/// is single-block (it must complete within one host call), so we loop over +/// `run` with a per-step budget until the translation reports a result. +/// +/// This budget also doubles as the quantization granularity of the synthetic +/// cost reported back to the pallet (see `consumed_cost_ps` below): `run` +/// doesn't return unused budget, so every completed step is charged in full. +/// 10ms keeps that over-approximation small relative to real translation +/// costs while still comfortably draining dev/undeployed-sized state in a +/// handful of iterations; the loop cap is a runaway backstop only. +const RUN_BUDGET_PS: u64 = 10_000_000_000; +const MAX_STEPS: usize = 100_000; + +/// Translate the ledger state referenced by a v8 arena root (`state_key_v8`, +/// the pallet's `StateKey` bytes) into a v9 ledger state, persist it, and +/// return the new v9 arena root (to store back into `StateKey`) together with +/// the synthetic cost, in picoseconds, the translation consumed against the +/// ledger's deterministic cost model — for the pallet to charge as this +/// migration's weight. +/// +/// If `state_key_v8` already references a ledger-9 state this is a no-op: it +/// returns the key unchanged with zero cost (see the idempotency guard below). +pub fn migrate_state_v8_to_v9( + state_key_v8: &[u8], +) -> Result<(Vec, u64), LedgerApiError> { + let t_total = std::time::Instant::now(); + + // 0. Idempotency guard: no-op if the state is already ledger-9. + // + // The pallet gates this behind `VersionedMigration<1, 2>`, but the on-chain + // pallet-midnight storage version is not a faithful proxy for the ledger + // version. The 2.0.0 runtime (spec 2_000_000) already runs ledger-9 yet + // shipped pallet-midnight at storage version 1 (it had no v1->v2 migration), + // so a network upgrading 2.0.0 -> this runtime still triggers this migration + // even though its `StateKey` already points at a v9 arena root. Feeding that + // v9 root to the v8 decode below would fail on the tag mismatch and abort the + // upgrade (the pallet `expect`s success), bricking the chain. Detect it by + // the root's serialized tag — `TypedArenaKey`'s tag embeds the `LedgerState` + // version (`storage-key(midnight:ledger-state[vN]:...)`), so a successful + // tagged decode as a v9 key is a reliable, arena-free discriminator — and + // return the key unchanged with zero synthetic cost. + if tagged_deserialize::, D::Hasher>>(&mut &state_key_v8[..]).is_ok() { + log::info!( + target: LOG_TARGET, + "StateKey already references a ledger-9 state; skipping v8->v9 translation (no-op)" + ); + return Ok((state_key_v8.to_vec(), 0)); + } + + // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. + let t_load = std::time::Instant::now(); + let key8: TypedArenaKey, D::Hasher> = tagged_deserialize(&mut &state_key_v8[..]) + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to deserialize v8 state key: {e:?}"); + LedgerApiError::Deserialization(DeserializationError::TypedArenaKey) + })?; + let ledger8: Sp, D> = default_storage::().arena.get_lazy(&key8).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to load v8 ledger from arena: {e:?}"); + LedgerApiError::NoLedgerState + })?; + log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 load took {:?}", t_load.elapsed()); + + // 2. Run the state translation table over the inner v8 `LedgerState`. + let t_translate = std::time::Instant::now(); + let input: Sp, D> = Sp::new(ledger8.state.clone()); + let mut tl = + TypedTranslationState::, LedgerState9, StateTranslationTable, D>::start( + input, + ) + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to start v8->v9 translation: {e:?}"); + LedgerApiError::HostApiError + })?; + + let run_budget = CostDuration::from_picoseconds(RUN_BUDGET_PS); + let mut steps = 0usize; + let state9: Sp, D> = loop { + steps += 1; + if steps > MAX_STEPS { + log::error!(target: LOG_TARGET, "v8->v9 translation did not converge in {MAX_STEPS} steps"); + return Err(LedgerApiError::HostApiError); + } + tl = tl.run(run_budget).map_err(|e| { + log::error!(target: LOG_TARGET, "v8->v9 translation step failed: {e:?}"); + LedgerApiError::HostApiError + })?; + if let Some(result) = tl.result().map_err(|e| { + log::error!(target: LOG_TARGET, "v8->v9 translation result failed: {e:?}"); + LedgerApiError::HostApiError + })? { + break result; + } + }; + // Every completed `run` call before the last one must have exhausted its + // full `RUN_BUDGET_PS` — otherwise `tl.result()` would already have been + // `Some` and the loop would have stopped there. Only the final call may + // have used less. `steps * RUN_BUDGET_PS` is therefore a deterministic + // upper bound on the synthetic cost actually consumed, accurate to one + // `RUN_BUDGET_PS` quantum. + let consumed_cost_ps = steps as u64 * RUN_BUDGET_PS; + log::info!( + target: LOG_TARGET, + "v8->v9 ledger state translation complete in {steps} step(s), {:?}, {consumed_cost_ps}ps synthetic cost", + t_translate.elapsed() + ); + + // 3. Wrap the translated state in the v9 ledger wrapper, persist it, and + // flush the arena so the new root is durable before the pallet stores it. + let t_persist = std::time::Instant::now(); + let ledger9 = Ledger9::new((*state9).clone()); + let mut sp9: Sp, D> = default_storage::().arena.alloc(ledger9); + sp9.persist(); + default_storage::().with_backend(|backend| backend.flush_all_changes_to_db()); + log::debug!( + target: LOG_TARGET, + "[perf] migrate_state_v8_to_v9 persist+flush took {:?}", + t_persist.elapsed() + ); + + // 4. Serialize the new v9 arena root for the pallet to store in `StateKey`. + let mut bytes = Vec::new(); + tagged_serialize(&sp9.as_typed_key(), &mut bytes).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to serialize v9 state key: {e:?}"); + LedgerApiError::Serialization(SerializationError::TypedArenaKey) + })?; + + log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 took {:?}", t_total.elapsed()); + Ok((bytes, consumed_cost_ps)) +} + +#[cfg(test)] +mod tests { + use super::*; + use ledger_storage_ledger_8::db::InMemoryDB; + + /// Dev-only: verify that seeding a v8 genesis blob with *this* crate's + /// ledger_8 `Ledger` wrapper reproduces the exact arena root a ledger-8 node + /// (e.g. release 1.0.1) stored in the chain-spec as `genesisStateKey`. If the + /// wrapper serialization drifted, the seeded root wouldn't match and block + /// execution after boot would fail to find the state. Runs only when the two + /// blob paths are provided via env (extracted from a fork-from chain-spec). + #[test] + fn v8_genesis_seed_root_matches_chainspec_key() { + let (Ok(gs_path), Ok(key_path)) = + (std::env::var("HF_GENESIS_STATE"), std::env::var("HF_GENESIS_KEY")) + else { + eprintln!("skipping: set HF_GENESIS_STATE and HF_GENESIS_KEY to run"); + return; + }; + let genesis = std::fs::read(gs_path).expect("read genesis_state"); + let expected = std::fs::read(key_path).expect("read genesisStateKey"); + + let state: LedgerState8 = + midnight_serialize::tagged_deserialize(&mut &genesis[..]) + .expect("deserialize v8 state"); + let ledger = Ledger8::::new(state); + let mut sp = default_storage::().arena.alloc(ledger); + sp.persist(); + let mut got = Vec::new(); + tagged_serialize(&sp.as_typed_key(), &mut got).expect("serialize key"); + + assert_eq!( + got, + expected, + "seeded v8 root must match the chain-spec genesisStateKey \n got={} \n exp={}", + hex::encode(&got), + hex::encode(&expected), + ); + } +} diff --git a/ledger/src/host_api/mod.rs b/ledger/src/host_api/mod.rs index 2b60cdd3a..1b1fd44c3 100644 --- a/ledger/src/host_api/mod.rs +++ b/ledger/src/host_api/mod.rs @@ -14,3 +14,7 @@ pub mod ledger_7; pub mod ledger_8; pub mod ledger_9; + +/// Host-side v8 -> v9 ledger state translation used by the runtime storage migration. +#[cfg(feature = "std")] +pub mod migration_8_to_9; diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index a62f0290a..c6c2787e6 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -154,6 +154,55 @@ pub fn drop_all_default_storage() { ledger_9::storage::drop_default_storage_if_exists(); } +/// Seed the (separate) ledger arena from a genesis `LedgerState` blob, using the +/// deserializer that matches the blob's `ledger-state[vN]` header tag. +/// +/// A node may boot on a chain-spec produced by an older runtime — notably the +/// ledger 8->9 hardfork, where a ledger-9 node starts from a ledger-8 +/// (`ledger-state[v13]`) genesis and only upgrades to v9 later via the runtime +/// migration. Seeding must therefore match the genesis version (the genesis +/// block runs under the old WASM and expects the old-format arena root), not the +/// latest. v8 and v9 share one storage backend, so a v8-seeded arena is exactly +/// what the post-migration v9 runtime reads. Unrecognized tags fall back to the +/// latest version (`ledger_9`), preserving the prior default behaviour. +#[cfg(feature = "std")] +pub fn init_ledger_storage_separate>( + dir: P, + genesis_state: &[u8], + cache_size: usize, +) -> alloc::vec::Vec { + if ledger_8::storage::genesis_matches_this_version(genesis_state) { + ledger_8::storage::init_storage_paritydb_separate(dir, genesis_state, cache_size) + } else { + ledger_9::storage::init_storage_paritydb_separate(dir, genesis_state, cache_size) + } +} + +/// Unified-DB counterpart of [`init_ledger_storage_separate`]. +#[cfg(feature = "std")] +pub fn init_ledger_storage_unified< + D: core::ops::Deref + Default + Send + Sync + 'static, + const COLUMN_OFFSET: u8, +>( + db_instance: D, + genesis_state: &[u8], + cache_size: usize, +) -> alloc::vec::Vec { + if ledger_8::storage::genesis_matches_this_version(genesis_state) { + ledger_8::storage::init_storage_paritydb_unified::( + db_instance, + genesis_state, + cache_size, + ) + } else { + ledger_9::storage::init_storage_paritydb_unified::( + db_instance, + genesis_state, + cache_size, + ) + } +} + mod common; pub mod types { diff --git a/ledger/src/versions/common/mod.rs b/ledger/src/versions/common/mod.rs index 270e71c5d..0c9b72588 100644 --- a/ledger/src/versions/common/mod.rs +++ b/ledger/src/versions/common/mod.rs @@ -1157,6 +1157,14 @@ where let system_tx = super::system_tx::unlock_to_treasury_system_tx(amount)?; api.tagged_serialize(&system_tx) } + + pub fn construct_distribute_treasury_system_tx( + amount: u128, + ) -> Result, LedgerApiError> { + let api = api::new(); + let system_tx = super::system_tx::distribute_treasury_system_tx(amount)?; + api.tagged_serialize(&system_tx) + } } #[cfg(feature = "std")] diff --git a/ledger/src/versions/common/storage.rs b/ledger/src/versions/common/storage.rs index ef39ea8d9..496757b52 100644 --- a/ledger/src/versions/common/storage.rs +++ b/ledger/src/versions/common/storage.rs @@ -74,6 +74,25 @@ impl core::fmt::Display for GetRootError { } } +/// Returns true if `genesis_state` is a tagged-serialized `LedgerState` of *this* +/// ledger version (i.e. its `midnight:ledger-state[vN]:` header tag matches this +/// version's `LedgerState::tag()`). +/// +/// Used to pick the correct version-specific seeder for the genesis arena when a +/// node boots on a chain-spec produced by an older runtime (e.g. the ledger 8->9 +/// hardfork, where a ledger-9 node starts from a ledger-8 `ledger-state[v13]` +/// genesis before the runtime upgrade migrates it to v9). +#[cfg(feature = "std")] +pub fn genesis_matches_this_version(genesis_state: &[u8]) -> bool { + use super::ledger_storage_local::DefaultDB; + let expected = as Tagged>::tag(); + // `peek_tag` reads the serialized header tag without deserializing the body. + match super::midnight_serialize_local::peek_tag(&mut std::io::Cursor::new(genesis_state)) { + Ok(tag) => tag.as_str() == expected.as_ref(), + Err(_) => false, + } +} + pub fn get_root(state: &[u8], network_id: Option<&str>) -> Result, GetRootError> { // Get empty state key use super::api::Ledger; diff --git a/ledger/src/versions/system_tx/ledger_7.rs b/ledger/src/versions/system_tx/ledger_7.rs index d6c5807c3..292832b0d 100644 --- a/ledger/src/versions/system_tx/ledger_7.rs +++ b/ledger/src/versions/system_tx/ledger_7.rs @@ -37,3 +37,8 @@ pub fn unlock_to_treasury_system_tx(_amount: u128) -> Result bool { false } + +/// Not applicable to ledger-7 (only the ledger-8 bridge exposes this host fn). +pub fn distribute_treasury_system_tx(_amount: u128) -> Result { + Err(LedgerApiError::HostApiError) +} diff --git a/ledger/src/versions/system_tx/ledger_8.rs b/ledger/src/versions/system_tx/ledger_8.rs index d6c5807c3..a893345eb 100644 --- a/ledger/src/versions/system_tx/ledger_8.rs +++ b/ledger/src/versions/system_tx/ledger_8.rs @@ -34,6 +34,14 @@ pub fn unlock_to_treasury_system_tx(_amount: u128) -> Result9 hardfork boundary — the ledger-8 runtime imports the corresponding +/// `construct_distribute_treasury_system_tx` host function (removed for v9). +pub fn distribute_treasury_system_tx(amount: u128) -> Result { + Ok(SystemTransaction::PayBlockRewardsToTreasury { amount }) +} + pub fn is_unlock_to_treasury_system_tx(_tx: &SystemTransaction) -> bool { false } diff --git a/ledger/src/versions/system_tx/ledger_9.rs b/ledger/src/versions/system_tx/ledger_9.rs index 6e854f4a3..99bd7a762 100644 --- a/ledger/src/versions/system_tx/ledger_9.rs +++ b/ledger/src/versions/system_tx/ledger_9.rs @@ -32,3 +32,10 @@ pub fn unlock_to_treasury_system_tx(amount: u128) -> Result bool { matches!(tx, SystemTransaction::UnlockToTreasury { .. }) } + +/// Not applicable to ledger-9: the block-rewards-to-treasury system tx was +/// removed for v9 (only the ledger-8 bridge exposes this host fn, so the ledger-8 +/// WASM can be executed across the 8->9 hardfork). +pub fn distribute_treasury_system_tx(_amount: u128) -> Result { + Err(LedgerApiError::HostApiError) +} diff --git a/local-environment/package-lock.json b/local-environment/package-lock.json index ce6bae903..9a0bead50 100644 --- a/local-environment/package-lock.json +++ b/local-environment/package-lock.json @@ -1552,15 +1552,15 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/call-bind-apply-helpers": { diff --git a/node/src/backend/custom_parity_db.rs b/node/src/backend/custom_parity_db.rs index 5cd5ab729..ea4693256 100644 --- a/node/src/backend/custom_parity_db.rs +++ b/node/src/backend/custom_parity_db.rs @@ -109,7 +109,10 @@ pub fn open>( match storage_config.separation { StorageSeparation::Separate => { - midnight_node_ledger::ledger_9::storage::init_storage_paritydb_separate( + // Version-aware: a ledger-9 node may boot on a ledger-8 genesis during + // the 8->9 hardfork; seed the arena with the deserializer matching the + // genesis `ledger-state[vN]` tag (see `init_ledger_storage_separate`). + midnight_node_ledger::init_ledger_storage_separate( &storage_config.db_path, &storage_config.genesis_state, storage_config.cache_size, @@ -117,10 +120,11 @@ pub fn open>( Ok((OwnedDb(db), LedgerStorageDb::SeparateDb(storage_config.db_path.clone()))) }, StorageSeparation::Unified => { - midnight_node_ledger::ledger_9::storage::init_storage_paritydb_unified::< - _, - NUM_COLUMNS_POLKADOT, - >(OwnedDb(db.clone()), &storage_config.genesis_state, storage_config.cache_size); + midnight_node_ledger::init_ledger_storage_unified::<_, NUM_COLUMNS_POLKADOT>( + OwnedDb(db.clone()), + &storage_config.genesis_state, + storage_config.cache_size, + ); Ok((OwnedDb(db.clone()), LedgerStorageDb::UnifiedDb(db.clone()))) }, } diff --git a/pallets/midnight/src/lib.rs b/pallets/midnight/src/lib.rs index 0fe085103..0529af6a2 100644 --- a/pallets/midnight/src/lib.rs +++ b/pallets/midnight/src/lib.rs @@ -99,7 +99,10 @@ pub mod pallet { } } - const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + // v2: ledger v8 -> v9 state translation (see `migrations::v2`). A ledger-8 + // runtime is at on-chain version 1; upgrading to this runtime runs the + // `MigrateV1ToV2` translation. Fresh ledger-9 genesis starts at version 2. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(2); // Manually add ~1% of block weight pub const EXTRA_WEIGHT_TX_SIZE: Weight = Weight::from_parts(20_000_000_000, 0); diff --git a/pallets/midnight/src/migrations/mod.rs b/pallets/midnight/src/migrations/mod.rs index bc3f4785d..c7a82f25e 100644 --- a/pallets/midnight/src/migrations/mod.rs +++ b/pallets/midnight/src/migrations/mod.rs @@ -23,3 +23,6 @@ pub const PALLET_MIGRATIONS_ID: &[u8; 19] = b"pallet-midnight-mbm"; // See https://github.com/input-output-hk/midnight-substrate-prototype/pull/382 // for the example of such a migration. // pub mod v1; + +/// Single-block ledger v8 -> v9 state translation (storage version 1 -> 2). +pub mod v2; diff --git a/pallets/midnight/src/migrations/v2.rs b/pallets/midnight/src/migrations/v2.rs new file mode 100644 index 000000000..e9154eca7 --- /dev/null +++ b/pallets/midnight/src/migrations/v2.rs @@ -0,0 +1,117 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage migration from v1 to v2: ledger-v8 -> ledger-v9 state translation. +//! +//! This is the on-chain half of the ledger 8 -> 9 hardfork. The pallet stores +//! only the ledger state's arena root in [`crate::StateKey`]; the `LedgerState` +//! itself lives in the ledger arena (parity-db). This migration hands the v8 +//! root to the [`migrate_state_v8_to_v9`](midnight_node_ledger::host_api::migration_8_to_9) +//! host function, which walks the v8 state, translates it into the v9 shape +//! (see [`midnight_node_ledger::state_translation_v8_to_v9`]), re-persists it, +//! and returns the new v9 root — which we write back into `StateKey`. +//! +//! It is a single-block migration: the host call translates the whole state in +//! one shot (a few milliseconds for dev/undeployed-sized state). It is wired +//! into [`crate::Migrations`](../../../runtime/src/lib.rs) via +//! [`VersionedMigration`], so it runs at most once, when a runtime at +//! pallet-midnight storage version 1 upgrades to this runtime (storage version +//! 2). A chain whose genesis is already ledger-9 starts at storage version 2 +//! and never runs it. +//! +//! Storage version 1 does not, however, imply the *ledger* state is still v8: +//! the 2.0.0 runtime already ran ledger-9 yet shipped pallet-midnight at storage +//! version 1 (it had no v1->v2 migration), so a network upgrading 2.0.0 -> this +//! runtime triggers this migration over an already-v9 state. The host function +//! detects that case and no-ops (returns the `StateKey` unchanged), so the only +//! effect on that path is the storage-version bump to 2. + +#[cfg(feature = "try-runtime")] +extern crate alloc; + +use crate::{Pallet, StateKey, pallet::Config}; +use frame_support::{ + migrations::VersionedMigration, pallet_prelude::*, traits::UncheckedOnRuntimeUpgrade, +}; +use midnight_node_ledger::types::active_ledger_bridge as LedgerApi; + +#[cfg(feature = "try-runtime")] +use alloc::vec::Vec; + +/// [`UncheckedOnRuntimeUpgrade`] implementation wrapped by [`MigrateV1ToV2`]. +pub struct InnerMigrateV1ToV2(core::marker::PhantomData); + +impl UncheckedOnRuntimeUpgrade for InnerMigrateV1ToV2 { + fn on_runtime_upgrade() -> Weight { + let state_key = StateKey::::get(); + + // The host function reads the v8 arena root, translates the referenced + // `LedgerState` to v9, re-persists it, and returns the new v9 root + // together with the synthetic cost (picoseconds) it charged the + // translation against the ledger's deterministic cost model. If the + // state is already v9 (e.g. a 2.0.0 -> this-runtime upgrade), it no-ops + // and returns the `state_key` unchanged with zero cost. + // A genuine failure here is unrecoverable: the chain would be left + // pointing at a v8 state a ledger-9 runtime cannot read, so we abort the + // upgrade. + let (new_state_key, consumed_cost_ps) = LedgerApi::migrate_state_v8_to_v9(&state_key) + .expect("FATAL: ledger v8->v9 state migration failed"); + + StateKey::::put(new_state_key); + log::info!( + target: "midnight::migration", + "ledger v8->v9 state migration complete; StateKey re-pointed to v9 root ({consumed_cost_ps}ps synthetic cost)" + ); + + // The translation runs natively in the host call, so the pallet-side + // read/write above is negligible next to it; report the ledger's own + // synthetic cost as `ref_time` instead. This is the same 1:1 mapping + // `pallet_midnight::get_tx_weight` uses for ordinary transactions + // (ledger picoseconds -> `Weight` ref_time), and that cost model + // already prices the arena reads/writes the translation performs, so + // no separate `DbWeight` charge is added on top. `proof_size` is 0: + // this chain doesn't build a PoV. Capped at the block's max weight, + // since a pathological state could in principle exceed it. + Weight::from_parts(consumed_cost_ps, 0).min(T::BlockWeights::get().max_block) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + frame_support::ensure!( + !StateKey::::get().is_empty(), + "ledger StateKey must be populated before v8->v9 migration" + ); + Ok(Vec::new()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + frame_support::ensure!( + !StateKey::::get().is_empty(), + "ledger StateKey must remain populated after v8->v9 migration" + ); + Ok(()) + } +} + +/// Translates the ledger state from v8 to v9 and bumps pallet-midnight storage +/// version 1 -> 2. Wired into the runtime `Migrations` tuple. +pub type MigrateV1ToV2 = VersionedMigration< + 1, + 2, + InnerMigrateV1ToV2, + Pallet, + ::DbWeight, +>; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 4fc222234..94ba7b4fb 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -1188,7 +1188,13 @@ pub type Executive = frame_executive::Executive< /// Extrinsic type that has already been checked. pub type CheckedExtrinsic = generic::CheckedExtrinsic; /// Migrations to apply on runtime upgrade. -pub type Migrations = (pallet_throttle::migrations::v1::MigrateV0ToV1,); +pub type Migrations = ( + pallet_throttle::migrations::v1::MigrateV0ToV1, + // Ledger v8 -> v9 state translation (the ledger 8->9 hardfork). Runs once, + // when a ledger-8 runtime (pallet-midnight storage version 1) upgrades to + // this ledger-9 runtime (storage version 2). + pallet_midnight::migrations::v2::MigrateV1ToV2, +); impl frame_system::offchain::CreateTransaction for Runtime where diff --git a/util/toolkit/src/commands/runtime_upgrade.rs b/util/toolkit/src/commands/runtime_upgrade.rs index 2d145167f..367b6daf2 100644 --- a/util/toolkit/src/commands/runtime_upgrade.rs +++ b/util/toolkit/src/commands/runtime_upgrade.rs @@ -14,9 +14,13 @@ // limitations under the License. use std::str::FromStr; +use std::time::Duration; use clap::Args; -use subxt::{OnlineClient, SubstrateConfig, dynamic}; +use subxt::{ + OnlineClient, SubstrateConfig, dynamic, + rpcs::{RpcClient, rpc_params}, +}; use thiserror::Error; use crate::commands::root_call::{self, RootCallArgs}; @@ -35,6 +39,10 @@ pub enum RuntimeUpgradeError { ExtrinsicError(#[from] subxt::error::ExtrinsicError), #[error("transaction finalized error: {0}")] TransactionFinalizedError(#[from] subxt::error::TransactionFinalizedSuccessError), + #[error("transaction progress error: {0}")] + TransactionProgressError(#[from] subxt::error::TransactionProgressError), + #[error("rpc error: {0}")] + RpcError(#[from] subxt::rpcs::Error), #[error("events error: {0}")] EventsError(#[from] subxt::error::EventsError), #[error("keypair parse error: {0}")] @@ -43,6 +51,41 @@ pub enum RuntimeUpgradeError { RootCallError(Box), #[error("runtime upgrade failed: CodeUpdated event not found")] CodeUpdateNotFound, + #[error("timed out waiting for the apply_authorized_upgrade transaction to finalize")] + ApplyFinalizeTimeout, + #[error("runtime upgrade did not enact: spec_version stayed at {0} after applying the code")] + UpgradeNotEnacted(u32), +} + +/// Query the on-chain runtime spec_version via the raw `state_getRuntimeVersion` RPC. +/// +/// We avoid subxt's typed metadata here on purpose: across a runtime upgrade the +/// client's metadata switches to the new runtime, so decoding anything encoded by +/// the old runtime (e.g. the `System.CodeUpdated` event in the apply block) is +/// unreliable. The raw JSON spec_version has no such dependency. +/// (finalized_height, spec_version at the finalized head). +/// +/// We track both because `state_getRuntimeVersion(finalized)` reports the code +/// *stored at* that block — which flips to the new runtime already at the +/// `apply_authorized_upgrade` block, even though that block *executed* under the +/// old runtime (its `MNSV` digest — how the toolkit fetcher classifies a block's +/// ledger version — is still the old spec). The first block that actually +/// executes the new runtime is `apply + 1`. So "spec flipped at the finalized +/// head" is one block early; callers must additionally wait for the finalized +/// height to advance past that point before the fetcher will see a ledger-9 block. +async fn finalized_state(rpc: &RpcClient) -> Result<(u64, u32), RuntimeUpgradeError> { + let hash: serde_json::Value = rpc.request("chain_getFinalizedHead", rpc_params![]).await?; + let header: serde_json::Value = + rpc.request("chain_getHeader", rpc_params![hash.clone()]).await?; + let version: serde_json::Value = + rpc.request("state_getRuntimeVersion", rpc_params![hash]).await?; + let height = header + .get("number") + .and_then(|n| n.as_str()) + .and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + .unwrap_or(0); + let spec = version.get("specVersion").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + Ok((height, spec)) } #[derive(Args)] @@ -78,7 +121,8 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError log::info!("Code hash: 0x{}", hex::encode(code_hash)); // Step 3: Build System::authorize_upgrade call and encode it - let api = OnlineClient::::from_insecure_url(&args.rpc_url).await?; + let rpc_client = RpcClient::from_insecure_url(&args.rpc_url).await?; + let api = OnlineClient::::from_rpc_client(rpc_client.clone()).await?; let authorize_upgrade_call = dynamic::tx("System", "authorize_upgrade", vec![dynamic::Value::from_bytes(&code_hash)]); let encoded_call = api.tx().await?.call_data(&authorize_upgrade_call)?; @@ -101,28 +145,54 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError let apply_upgrade_call = dynamic::tx("System", "apply_authorized_upgrade", vec![dynamic::Value::from_bytes(&code)]); - let apply_events = api - .tx() - .await? - .sign_and_submit_then_watch_default(&apply_upgrade_call, &signer) - .await? - .wait_for_finalized_success() - .await?; - - // Step 6: Verify CodeUpdated event - let mut success = false; - for event in apply_events.iter() { - let event = event?; - if event.pallet_name() == "System" && event.event_name() == "CodeUpdated" { - log::info!("Code update success: {:?}", event); - success = true; - break; + let (_, pre_spec_version) = finalized_state(&rpc_client).await?; + log::info!("Pre-upgrade spec_version (finalized): {pre_spec_version}"); + + // Wait only for finalization — NOT `wait_for_finalized_success`, which eagerly + // decodes the block's events. `apply_authorized_upgrade` swaps the on-chain + // code, and subxt's metadata follows it to the new runtime, so decoding the + // old runtime's `System.CodeUpdated` event in that block fails. Confirm the + // upgrade succeeded by observing the spec_version bump below instead. + let submit = async { + api.tx() + .await? + .sign_and_submit_then_watch_default(&apply_upgrade_call, &signer) + .await? + .wait_for_finalized() + .await + .map_err(RuntimeUpgradeError::from) + }; + tokio::time::timeout(Duration::from_secs(120), submit) + .await + .map_err(|_| RuntimeUpgradeError::ApplyFinalizeTimeout)??; + + // Step 6: Confirm the upgrade is not just applied but *executing* at a + // finalized block. `state_getRuntimeVersion(finalized)` reports the stored + // code, which flips to the new runtime already at the apply block — but that + // block's `MNSV` execution-version digest (how the fetcher classifies it) is + // still the old spec. The first block that runs the new runtime is apply+1. + // So: note the finalized height where the stored spec first exceeds pre, then + // wait for the finalized height to advance past it (apply+1 finalized). Only + // then will a downstream `fetch` see a ledger-9-classified block. + let mut flip_height: Option = None; + for _ in 0..60 { + tokio::time::sleep(Duration::from_secs(3)).await; + let (height, spec) = finalized_state(&rpc_client).await?; + if spec > pre_spec_version { + match flip_height { + None => flip_height = Some(height), + Some(h0) if height > h0 => { + log::info!( + "Runtime upgrade completed successfully! spec_version {pre_spec_version} -> {spec}; \ + new runtime executing since finalized #{}, now finalized #{height}", + h0 + 1, + ); + return Ok(()); + }, + _ => {}, + } } } - if !success { - return Err(RuntimeUpgradeError::CodeUpdateNotFound); - } - log::info!("Runtime upgrade completed successfully!"); - Ok(()) + Err(RuntimeUpgradeError::UpgradeNotEnacted(pre_spec_version)) } diff --git a/util/toolkit/src/tx_generator/builder/mod.rs b/util/toolkit/src/tx_generator/builder/mod.rs index 372cb2884..ade6566f7 100644 --- a/util/toolkit/src/tx_generator/builder/mod.rs +++ b/util/toolkit/src/tx_generator/builder/mod.rs @@ -19,7 +19,7 @@ use midnight_node_ledger_helpers::fork::{ fork_aware_context::{ ForkAwareLedgerContext, apply_block_7, apply_block_8, apply_block_9, block_context_from_raw_7, block_context_from_raw_8, block_context_from_raw_9, - fork_context_7_to_8, + fork_context_7_to_8, fork_context_8_to_9, }, raw_block_data::{LedgerVersion, RawBlockData}, }; @@ -1334,6 +1334,25 @@ fn replay_blocks_9( } } +/// Fork a ledger-8 context to ledger 9 (real state translation) and replay the +/// ledger-9 blocks, if any. Returns the ledger-8 context unchanged when there are +/// no ledger-9 blocks. +fn fork_8_to_9_if_needed( + ctx8: midnight_node_ledger_helpers::ledger_8::context::LedgerContext, + l9_blocks: &[RawBlockData], + cached: &[(WalletSeed, CachedWalletState)], + schemes: &WalletSchemes, +) -> ForkAwareLedgerContext { + if l9_blocks.is_empty() { + ForkAwareLedgerContext::Ledger8(ctx8) + } else { + let ctx9 = + timed!("fork_context_8_to_9", fork_context_8_to_9(ctx8)).expect("fork 8 to 9 failed"); + replay_blocks_9(&ctx9, l9_blocks, cached, schemes); + ForkAwareLedgerContext::Ledger9(ctx9) + } +} + /// Replays blocks across a potential Ledger7→Ledger8->Ledger9 fork boundaries, /// injecting cached wallets at their saved height. pub(crate) fn replay_blocks( @@ -1358,27 +1377,27 @@ pub(crate) fn replay_blocks( l8_and_l9_blocks.partition_point(|b| b.ledger_version() == LedgerVersion::Ledger8); let (l8_blocks, l9_blocks) = l8_and_l9_blocks.split_at(fork_8_to_9_idx); - assert!( - l9_blocks.is_empty() || (l7_blocks.is_empty() && l8_blocks.is_empty()), - "chain has Ledger9 blocks and eariler version blocks. This is not supported yet!" - ); - + // Replay each version's blocks in order, forking the context across the + // 7->8 and 8->9 boundaries as needed. The 8->9 fork performs a real state + // translation (see `fork_context_8_to_9`) so post-hardfork transactions are + // built at ledger 9, matching the upgraded chain. let result = match fork_ctx { ForkAwareLedgerContext::Ledger7(ctx7) => { replay_blocks_7(&ctx7, l7_blocks); - if l8_blocks.is_empty() { - assert!(cached.is_empty(), "cached wallets with no Ledger8 blocks"); + if l8_blocks.is_empty() && l9_blocks.is_empty() { + assert!(cached.is_empty(), "cached wallets with no Ledger8/9 blocks"); ForkAwareLedgerContext::Ledger7(ctx7) } else { - let ctx8 = fork_context_7_to_8(ctx7).expect("fork 7 to 8 failed"); + let ctx8 = timed!("fork_context_7_to_8", fork_context_7_to_8(ctx7)) + .expect("fork 7 to 8 failed"); replay_blocks_8(&ctx8, l8_blocks); - ForkAwareLedgerContext::Ledger8(ctx8) + fork_8_to_9_if_needed(ctx8, l9_blocks, cached, schemes) } }, ForkAwareLedgerContext::Ledger8(ctx8) => { assert!(l7_blocks.is_empty(), "Ledger7 blocks with Ledger8 context"); replay_blocks_8(&ctx8, l8_blocks); - ForkAwareLedgerContext::Ledger8(ctx8) + fork_8_to_9_if_needed(ctx8, l9_blocks, cached, schemes) }, ForkAwareLedgerContext::Ledger9(ctx9) => { assert!(l7_blocks.is_empty(), "Ledger7 blocks with Ledger9 context"); diff --git a/util/toolkit/test-images.docker-compose.yml b/util/toolkit/test-images.docker-compose.yml index 42cdd834f..71a079559 100644 --- a/util/toolkit/test-images.docker-compose.yml +++ b/util/toolkit/test-images.docker-compose.yml @@ -15,8 +15,11 @@ services: depends_on: guard: condition: service_completed_successfully + # Ledger-8 release: the hardfork_e2e test forks from this (ledger 8, runtime + # spec_version 1_000_000, pallet-midnight storage version 1) up to the current + # ledger-9 runtime, exercising the v8->v9 state migration. midnight-node-fork-from: - image: ${FORK_FROM_NODE_IMAGE:-midnightntwrk/midnight-node:0.21.0} + image: ${FORK_FROM_NODE_IMAGE:-midnightntwrk/midnight-node:1.0.1} depends_on: guard: condition: service_completed_successfully diff --git a/util/toolkit/tests/hardfork_e2e.rs b/util/toolkit/tests/hardfork_e2e.rs index 578788807..8507a90ac 100644 --- a/util/toolkit/tests/hardfork_e2e.rs +++ b/util/toolkit/tests/hardfork_e2e.rs @@ -54,7 +54,6 @@ async fn run_cli(args: &[&str]) { } #[test_log::test(tokio::test)] -#[ignore = "Migration to Ledger v9 is not yet supported. Issue #1580."] async fn hardfork_single_tx() { // 1. Generate chain-spec from fork-from node let (old_name, old_tag) = test_image("midnight-node-fork-from");