From 9fce1486fccd8abeb11895aa488c8b01f81fe8e3 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:57:06 +0100 Subject: [PATCH 01/19] feat(ledger): add v8->v9 storage migration for the ledger 8->9 hardfork Port the v8->v9 state translation table from midnight-ledger PR #539 into `midnight_node_ledger::state_translation_v8_to_v9`, and wire it into the runtime as a single-block storage migration: - New host function `Ledger9Bridge::migrate_state_v8_to_v9` reads the v8 arena root (pallet-midnight `StateKey`), walks/translates the v8 `LedgerState` into the v9 shape, re-persists it, and returns the new v9 root. v8 and v9 share one storage backend, so the arena is shared. - `pallet_midnight::migrations::v2::MigrateV1ToV2` (VersionedMigration 1->2) calls the host function and re-points `StateKey`. Bumped pallet-midnight STORAGE_VERSION 1 -> 2 and added it to the runtime `Migrations` tuple. - Un-ignore the `hardfork_single_tx` e2e test and point its fork-from image at the ledger-8 release `midnightntwrk/midnight-node:1.0.1`. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- Cargo.lock | 4 + Cargo.toml | 11 +- ledger/Cargo.toml | 4 + ledger/src/host_api/ledger_9.rs | 23 + ledger/src/host_api/migration_8_to_9.rs | 124 ++++ ledger/src/host_api/mod.rs | 4 + ledger/src/lib.rs | 5 + ledger/src/state_translation_v8_to_v9.rs | 637 ++++++++++++++++++++ pallets/midnight/src/lib.rs | 5 +- pallets/midnight/src/migrations/mod.rs | 3 + pallets/midnight/src/migrations/v2.rs | 98 +++ runtime/src/lib.rs | 8 +- util/toolkit/test-images.docker-compose.yml | 5 +- util/toolkit/tests/hardfork_e2e.rs | 1 - 14 files changed, 927 insertions(+), 5 deletions(-) create mode 100644 ledger/src/host_api/migration_8_to_9.rs create mode 100644 ledger/src/state_translation_v8_to_v9.rs create mode 100644 pallets/midnight/src/migrations/v2.rs diff --git a/Cargo.lock b/Cargo.lock index ae3325b4e..95cb3ade5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8065,6 +8065,8 @@ dependencies = [ "midnight-onchain-runtime 2.0.1", "midnight-onchain-runtime 3.1.0", "midnight-onchain-runtime 4.0.0", + "midnight-onchain-state 3.0.0", + "midnight-onchain-state 4.0.0", "midnight-primitives-ledger", "midnight-serialize", "midnight-storage 1.1.1", @@ -8734,6 +8736,7 @@ checksum = "210e601a89aee79ce2b007cf96c1a56c23baa306b0c40b9b98d12c27e18d1981" dependencies = [ "crypto", "derive-where", + "hashbrown 0.16.1", "midnight-base-crypto", "midnight-serialize", "midnight-storage-core", @@ -8755,6 +8758,7 @@ dependencies = [ "crypto", "derive-where", "fake", + "hashbrown 0.16.1", "hex", "itertools 0.14.0", "konst 0.4.3", diff --git a/Cargo.toml b/Cargo.toml index 97892b688..35483639c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,8 +91,14 @@ zkir = { version = "^2.2.0", package = "midnight-zkir" } # Ledger 8 (compatible with layout-v2) # coin-structure and transient-crypto (2.x) share versions with L7 so reuse those entries. mn-ledger-8 = { version = "=8.1.0", package = "midnight-ledger" } -ledger-storage-ledger-8 = { version = "=2.0.1", package = "midnight-storage", features = ["parity-db"] } +# `state-translation` (needed by the v8->v9 storage migration) pulls in +# `public-internal-structure`, exposing `merkle_patricia_trie`/`storable`/the +# `state_translation` module used by `midnight_node_ledger::state_translation_v8_to_v9`. +ledger-storage-ledger-8 = { version = "=2.0.1", package = "midnight-storage", features = ["parity-db", "state-translation"] } onchain-runtime-ledger-8 = { version = "=3.1.0", package = "midnight-onchain-runtime" } +# Same `midnight-onchain-state` instance that `mn-ledger-8`'s `ContractState` +# resolves to (via onchain-runtime 3.1.0); used as the v8 side of the state translation table. +onchain-state-ledger-8 = { version = "=3.0.0", package = "midnight-onchain-state" } zswap-ledger-8 = { version = "=8.1.0", package = "midnight-zswap" } # Ledger 9 (compatible with layout-v2; midnight-ledger-v9 crate) @@ -101,6 +107,9 @@ zswap-ledger-8 = { version = "=8.1.0", package = "midnight-zswap" } # (midnight-zkir 2.2.0) serves all of L7/L8/L9. mn-ledger-9 = { version = "=1.0.0", package = "midnight-ledger-v9" } onchain-runtime-ledger-9 = { version = "=4.0.0", package = "midnight-onchain-runtime" } +# v9 side of the state translation table (matches `mn-ledger-9`'s `ContractState` +# via onchain-runtime 4.0.0). Patched to onchain-state-4.0.0-rc.3 by [patch.crates-io]. +onchain-state-ledger-9 = { version = "=4.0.0", package = "midnight-onchain-state" } zswap-ledger-9 = { version = "=9.0.0", package = "midnight-zswap" } coin-structure-ledger-9 = { version = "=3.0.0", package = "midnight-coin-structure" } transient-crypto-ledger-9 = { version = "=3.0.0", package = "midnight-transient-crypto" } diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 4cbea9a5f..fe10434e4 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -18,11 +18,13 @@ zswap = { workspace = true, optional = true } mn-ledger-8 = { workspace = true, features = ["proving"], optional = true } onchain-runtime-ledger-8 = { workspace = true, optional = true } +onchain-state-ledger-8 = { workspace = true, optional = true } ledger-storage-ledger-8 = { workspace = true, optional = true } zswap-ledger-8 = { workspace = true, optional = true } mn-ledger-9 = { workspace = true, features = ["proving"], optional = true } onchain-runtime-ledger-9 = { workspace = true, optional = true } +onchain-state-ledger-9 = { workspace = true, optional = true } zswap-ledger-9 = { workspace = true, optional = true } coin-structure-ledger-9 = { workspace = true, optional = true } transient-crypto-ledger-9 = { workspace = true, optional = true } @@ -81,10 +83,12 @@ std = [ "zswap", "mn-ledger-8", "onchain-runtime-ledger-8", + "onchain-state-ledger-8", "ledger-storage-ledger-8", "zswap-ledger-8", "mn-ledger-9", "onchain-runtime-ledger-9", + "onchain-state-ledger-9", "zswap-ledger-9", "coin-structure-ledger-9", "transient-crypto-ledger-9", diff --git a/ledger/src/host_api/ledger_9.rs b/ledger/src/host_api/ledger_9.rs index 35bf5617b..ba064a147 100644 --- a/ledger/src/host_api/ledger_9.rs +++ b/ledger/src/host_api/ledger_9.rs @@ -501,6 +501,29 @@ pub trait Ledger9Bridge { true } + /// Translate the ledger state from ledger-v8 format to ledger-v9 format. + /// + /// Called by `pallet_midnight`'s v8->v9 storage migration during the runtime + /// upgrade that crosses into ledger-9. `state_key` is the pallet's `StateKey` + /// (a v8 arena root); the returned bytes are the new v9 arena root to store back. + fn migrate_state_v8_to_v9( + &mut self, + state_key: PassFatPointerAndRead<&[u8]>, + ) -> AllocateAndReturnByCodec, LedgerApiError>> { + // Ensure the ledger arena is initialized before translating. The migration + // runs in the Executive migrations tuple, before pallet_midnight's + // on_initialize/on_runtime_upgrade have (re)initialized storage this block. + // `set_default_storage` is idempotent — a no-op if the pre-fork ledger-8 + // blocks already set it (v8 and v9 share the same storage backend). + if is_unified(*self) { + Bridge::::set_default_storage(*self); + crate::host_api::migration_8_to_9::migrate_state_v8_to_v9::(state_key) + } else { + Bridge::::set_default_storage(*self); + crate::host_api::migration_8_to_9::migrate_state_v8_to_v9::(state_key) + } + } + /// Initialize a process-wide temporary ledger ParityDb seeded with the /// undeployed-network genesis state. /// diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs new file mode 100644 index 000000000..7392b56dd --- /dev/null +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -0,0 +1,124 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Host-side v8 -> v9 ledger state translation, driven by +//! [`crate::state_translation_v8_to_v9::StateTranslationTable`]. +//! +//! The on-chain pallet stores only the arena root of the ledger state +//! (`pallet_midnight::StateKey`, a `tagged_serialize`d `TypedArenaKey`); +//! the `LedgerState` itself lives in the process-global ledger arena (parity-db). +//! When the runtime upgrades from a ledger-8 runtime (spec < 2_000_000) to a +//! ledger-9 runtime (spec >= 2_000_000), the on-chain migration +//! (`pallet_midnight::migrations`) calls [`Ledger9Bridge::migrate_state_v8_to_v9`] +//! which lands here: it reads the v8 arena root, walks the v8 `LedgerState` +//! translating it into a v9 `LedgerState`, re-persists it, and returns the new v9 +//! arena root for the pallet to store back into `StateKey`. +//! +//! v8 and v9 share one storage crate (`ledger-storage-ledger-8`, +//! midnight-storage 2.0.1) and hence one arena, so the translation reads and +//! writes the same parity-db instance the pre-fork ledger-8 blocks populated. + +use crate::ledger_8::api::Ledger as Ledger8; +use crate::ledger_9::api::Ledger as Ledger9; +use crate::ledger_9::types::{DeserializationError, LedgerApiError, SerializationError}; +use crate::state_translation_v8_to_v9::StateTranslationTable; + +use base_crypto::cost_model::CostDuration; +use ledger_storage_ledger_8 as storage; +use midnight_serialize::{tagged_deserialize, tagged_serialize}; +use storage::{ + arena::{Sp, TypedArenaKey}, + db::DB, + state_translation::TypedTranslationState, + storage::default_storage, +}; + +type LedgerState8 = mn_ledger_8::structure::LedgerState; +type LedgerState9 = mn_ledger_9::structure::LedgerState; + +const LOG_TARGET: &str = "midnight::ledger::migration_8_to_9"; + +/// Picoseconds granted to each `TypedTranslationState::run` step. The migration +/// is single-block (it must complete within one host call), so we loop over +/// `run` with a generous per-step budget until the translation reports a result. +/// One second of picoseconds per step comfortably drains dev/undeployed-sized +/// state in a couple of iterations; the loop cap is a runaway backstop only. +const RUN_BUDGET_PS: u64 = 1_000_000_000_000; +const MAX_STEPS: usize = 100_000; + +/// Translate the ledger state referenced by a v8 arena root (`state_key_v8`, +/// the pallet's `StateKey` bytes) into a v9 ledger state, persist it, and return +/// the new v9 arena root to store back into `StateKey`. +pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, LedgerApiError> { + // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. + let key8: TypedArenaKey, D::Hasher> = + tagged_deserialize(&mut &state_key_v8[..]).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to deserialize v8 state key: {e:?}"); + LedgerApiError::Deserialization(DeserializationError::TypedArenaKey) + })?; + let ledger8: Sp, D> = default_storage::().arena.get_lazy(&key8).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to load v8 ledger from arena: {e:?}"); + LedgerApiError::NoLedgerState + })?; + + // 2. Run the state translation table over the inner v8 `LedgerState`. + let input: Sp, D> = Sp::new(ledger8.state.clone()); + let mut tl = TypedTranslationState::< + LedgerState8, + LedgerState9, + StateTranslationTable, + D, + >::start(input) + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to start v8->v9 translation: {e:?}"); + LedgerApiError::HostApiError + })?; + + let run_budget = CostDuration::from_picoseconds(RUN_BUDGET_PS); + let mut steps = 0usize; + let state9: Sp, D> = loop { + steps += 1; + if steps > MAX_STEPS { + log::error!(target: LOG_TARGET, "v8->v9 translation did not converge in {MAX_STEPS} steps"); + return Err(LedgerApiError::HostApiError); + } + tl = tl.run(run_budget).map_err(|e| { + log::error!(target: LOG_TARGET, "v8->v9 translation step failed: {e:?}"); + LedgerApiError::HostApiError + })?; + if let Some(result) = tl.result().map_err(|e| { + log::error!(target: LOG_TARGET, "v8->v9 translation result failed: {e:?}"); + LedgerApiError::HostApiError + })? { + break result; + } + }; + log::info!(target: LOG_TARGET, "v8->v9 ledger state translation complete in {steps} step(s)"); + + // 3. Wrap the translated state in the v9 ledger wrapper, persist it, and + // flush the arena so the new root is durable before the pallet stores it. + let ledger9 = Ledger9::new((*state9).clone()); + let mut sp9: Sp, D> = default_storage::().arena.alloc(ledger9); + sp9.persist(); + default_storage::().with_backend(|backend| backend.flush_all_changes_to_db()); + + // 4. Serialize the new v9 arena root for the pallet to store in `StateKey`. + let mut bytes = Vec::new(); + tagged_serialize(&sp9.as_typed_key(), &mut bytes).map_err(|e| { + log::error!(target: LOG_TARGET, "failed to serialize v9 state key: {e:?}"); + LedgerApiError::Serialization(SerializationError::TypedArenaKey) + })?; + Ok(bytes) +} diff --git a/ledger/src/host_api/mod.rs b/ledger/src/host_api/mod.rs index 2b60cdd3a..1b1fd44c3 100644 --- a/ledger/src/host_api/mod.rs +++ b/ledger/src/host_api/mod.rs @@ -14,3 +14,7 @@ pub mod ledger_7; pub mod ledger_8; pub mod ledger_9; + +/// Host-side v8 -> v9 ledger state translation used by the runtime storage migration. +#[cfg(feature = "std")] +pub mod migration_8_to_9; diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index a62f0290a..da1aba91a 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -27,6 +27,11 @@ pub mod json; #[cfg(feature = "std")] mod utils; +/// v8 -> v9 ledger state translation table (ported from midnight-ledger PR #539), +/// consumed by the v8->v9 storage migration host function. +#[cfg(feature = "std")] +pub mod state_translation_v8_to_v9; + pub mod host_api; #[path = "versions"] diff --git a/ledger/src/state_translation_v8_to_v9.rs b/ledger/src/state_translation_v8_to_v9.rs new file mode 100644 index 000000000..9c0523d42 --- /dev/null +++ b/ledger/src/state_translation_v8_to_v9.rs @@ -0,0 +1,637 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! State translation from ledger v8 to ledger v9. +//! +//! Ported from `midnight-ledger` PR #539 (`v8-to-v9-state-translation`). The +//! only changes from the upstream crate are the import aliases below, which map +//! the translation's `ledger_v8` / `ledger_v9` / `onchain_state_v8` / +//! `onchain_state_v9` / `storage` / `serialize` crate names onto this +//! workspace's package aliases. The [`StateTranslationTable`] is consumed by the +//! v8->v9 storage migration ([`crate::host_api::migration_8_to_9`]). +//! +//! ## State shape differences (only stored types listed) +//! +//! | type | v8 tag | v9 tag | change | +//! | ---------------------------- | ------------------------------------ | ------------------------------------ | ------ | +//! | LedgerState | `ledger-state[v13]` | `ledger-state[v18]` | `bridge_receiving` map gains `NightAnn` | +//! | LedgerParameters | `ledger-parameters[v5]` | `ledger-parameters[v8]` | adds `min_block_price`; `TransactionLimits` adds `max_contract_metadata_size`; `TransactionCostModel` drops `parallelism_factor`, adds `validation`/`guaranteed`/`fallible` factors | +//! | ContractState | `contract-state[v6]` | `contract-state[v8]` | reflows `ContractOperation` + `ContractMaintenanceAuthority` changes | +//! | ContractOperation | `contract-operation[v4]` | `contract-operation[v6]` | single `v2` key -> `{ v2, v3, ir }`; v8 key maps to `v2`, new `v3`/`ir` empty | +//! | ContractMaintenanceAuthority | `contract-maintenance-authority[v1]` | `contract-maintenance-authority[v2]` | `committee: Vec` -> `Vec` (Schnorr/ECDSA sum) | +//! +//! Everything else (zswap, utxo, dust, replay_protection, treasury, +//! unclaimed_block_rewards) is tag-stable and passes through `recast`. + +// Map the upstream translation crate names onto the node workspace's package +// aliases. `mn-ledger-8`/`mn-ledger-9` are the two `midnight-ledger` majors; +// `onchain-state-ledger-8`/`-9` are the exact `midnight-onchain-state` instances +// that each ledger's `ContractState` resolves to; `ledger-storage-ledger-8` +// (midnight-storage 2.0.1, `state-translation` feature) backs both. +use ledger_storage_ledger_8 as storage; +use midnight_serialize as serialize; +use mn_ledger_8 as ledger_v8; +use mn_ledger_9 as ledger_v9; +use onchain_state_ledger_8 as onchain_state_v8; +use onchain_state_ledger_9 as onchain_state_v9; + +use base_crypto::cost_model::CostDuration; +use serialize::Tagged; +use std::ops::Deref; +use std::{any::Any, borrow::Cow, io, marker::PhantomData}; +use storage::{ + Storable, + arena::Sp, + db::DB, + merkle_patricia_trie::{self, Annotation, MerklePatriciaTrie}, + state_translation::*, + storable::SizeAnn, + storage::{HashMap, Map, default_storage}, +}; + +// ---------- Generic helpers (copied from the v6->v7 reference) ---------- + +/// Recast a stored object from one type to another, requiring matching tags. +/// Used for subtrees whose tag is unchanged between v8 and v9. +fn recast + Tagged, B: Storable + Tagged, D: DB>( + a: &Sp, +) -> io::Result> { + if A::tag() != B::tag() { + return io::Result::Err(io::Error::new(io::ErrorKind::Other, "tags do not match")); + } + default_storage::().get_lazy(&a.as_child().into()) +} + +/// Generic MPT translation: walks the trie, translating each entry via the +/// table-registered translation for `A->B`, and recomputes annotations under +/// `AnnB` from the new values. +struct MptTl(PhantomData<(A, B, AnnA, AnnB)>); + +impl< + A: Storable + Tagged, + B: Storable + Tagged, + AnnA: Annotation + Storable + Tagged, + AnnB: Annotation + Storable + Tagged, + D: DB, +> DirectTranslation, MerklePatriciaTrie, D> + for MptTl +{ + fn required_translations() -> Vec { + vec![TranslationId( + merkle_patricia_trie::Node::::tag(), + merkle_patricia_trie::Node::::tag(), + )] + } + fn child_translations( + source: &MerklePatriciaTrie, + ) -> Vec<(TranslationId, Sp)> { + let tlids = , _, D>>::required_translations(); + vec![(tlids[0].clone(), source.0.upcast())] + } + fn finalize( + source: &MerklePatriciaTrie, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let tls = Self::child_translations(source); + Ok(Some(MerklePatriciaTrie(try_resopt!( + cache.resolve(&tls[0].0, tls[0].1.as_child()) + )))) + } +} + +impl< + A: Storable + Tagged, + B: Storable + Tagged, + AnnA: Storable + Tagged + Annotation, + AnnB: Storable + Tagged + Annotation, + D: DB, +> + DirectTranslation< + merkle_patricia_trie::Node, + merkle_patricia_trie::Node, + D, + > for MptTl +{ + fn required_translations() -> Vec { + let entry_tl = TranslationId(A::tag(), B::tag()); + let self_tl = TranslationId( + merkle_patricia_trie::Node::::tag(), + merkle_patricia_trie::Node::::tag(), + ); + vec![entry_tl, self_tl] + } + fn child_translations( + source: &merkle_patricia_trie::Node, + ) -> Vec<(TranslationId, Sp)> { + let tls = , _, D>>::required_translations(); + let entry_tl = tls[0].clone(); + let self_tl = tls[1].clone(); + match source { + merkle_patricia_trie::Node::Empty => vec![], + merkle_patricia_trie::Node::Branch { children, .. } => children + .iter() + .map(|child| (self_tl.clone(), child.upcast())) + .collect(), + merkle_patricia_trie::Node::Extension { child, .. } => { + vec![(self_tl, child.upcast())] + } + merkle_patricia_trie::Node::MidBranchLeaf { value, child, .. } => { + vec![(entry_tl, value.upcast()), (self_tl, child.upcast())] + } + merkle_patricia_trie::Node::Leaf { value, .. } => vec![(entry_tl, value.upcast())], + } + } + fn finalize( + source: &merkle_patricia_trie::Node, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let tls = Self::child_translations(source); + Ok(Some(match source { + merkle_patricia_trie::Node::Empty => merkle_patricia_trie::Node::Empty, + merkle_patricia_trie::Node::Branch { .. } => { + let mut new_children = + core::array::from_fn(|_| Sp::new(merkle_patricia_trie::Node::Empty)); + for (child, new_child) in tls.iter().zip(new_children.iter_mut()) { + *new_child = try_resopt!(cache.resolve(&child.0, child.1.as_child())); + } + let ann = new_children + .iter() + .fold(AnnB::empty(), |acc, x| { + acc.append(&merkle_patricia_trie::Node::::ann(x)) + }); + merkle_patricia_trie::Node::Branch { + ann, + children: Box::new(new_children), + } + } + merkle_patricia_trie::Node::Extension { + compressed_path, .. + } => { + let child: Sp, D> = + try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let ann = merkle_patricia_trie::Node::::ann(&child); + merkle_patricia_trie::Node::Extension { + ann, + compressed_path: compressed_path.clone(), + child, + } + } + merkle_patricia_trie::Node::Leaf { .. } => { + let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let ann = AnnB::from_value(&value); + merkle_patricia_trie::Node::Leaf { ann, value } + } + merkle_patricia_trie::Node::MidBranchLeaf { .. } => { + let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); + let child: Sp, D> = + try_resopt!(cache.resolve(&tls[1].0, tls[1].1.as_child())); + let ann = AnnB::from_value(&value) + .append(&merkle_patricia_trie::Node::::ann(&child)); + merkle_patricia_trie::Node::MidBranchLeaf { ann, value, child } + } + })) + } +} + +/// Identity translation for a type whose serialization is unchanged across +/// versions. Needed when an MPT's entries are tag-stable but its annotation +/// changes (e.g. `bridge_receiving`). +struct IdentityTl(PhantomData); + +impl + Clone, D: DB> DirectTranslation for IdentityTl { + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations(_: &T) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &T, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result> { + Ok(Some(source.clone())) + } +} + +// ---------- Translation IDs (shorthand) ---------- + +struct Ids; + +impl Ids { + fn contract_mpt() -> TranslationId { + TranslationId( + MerklePatriciaTrie::< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >::tag(), + MerklePatriciaTrie::< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >::tag(), + ) + } + + fn bridge_receiving_mpt() -> TranslationId { + TranslationId( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ) + } + + fn parameters() -> TranslationId { + TranslationId( + ledger_v8::structure::LedgerParameters::tag(), + ledger_v9::structure::LedgerParameters::tag(), + ) + } +} + +// ---------- Top-level: LedgerState v8 -> v9 ---------- + +struct LedgerStateTl; + +impl + DirectTranslation, ledger_v9::structure::LedgerState, D> + for LedgerStateTl +{ + fn required_translations() -> Vec { + vec![ + Ids::parameters::(), + Ids::bridge_receiving_mpt::(), + Ids::contract_mpt::(), + ] + } + + fn child_translations( + source: &ledger_v8::structure::LedgerState, + ) -> Vec<(TranslationId, Sp)> { + vec![ + (Ids::parameters::(), source.parameters.upcast()), + ( + Ids::bridge_receiving_mpt::(), + source.bridge_receiving.mpt.upcast(), + ), + (Ids::contract_mpt::(), source.contract.mpt.upcast()), + ] + } + + fn finalize( + source: &ledger_v8::structure::LedgerState, + _limit: &mut CostDuration, + cache: &TranslationCache, + ) -> io::Result>> { + let Some(parameters) = cache.lookup(&Ids::parameters::(), source.parameters.as_child()) + else { + return Ok(None); + }; + let Some(bridge_recv_mpt) = + cache.lookup(&Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.as_child()) + else { + return Ok(None); + }; + let Some(contract_mpt) = + cache.lookup(&Ids::contract_mpt::(), source.contract.mpt.as_child()) + else { + return Ok(None); + }; + + Ok(Some(ledger_v9::structure::LedgerState { + network_id: source.network_id.clone(), + parameters: parameters.force_downcast(), + locked_pool: source.locked_pool, + bridge_receiving: Map { + mpt: bridge_recv_mpt.force_downcast(), + key_type: PhantomData, + }, + reserve_pool: source.reserve_pool, + block_reward_pool: source.block_reward_pool, + unclaimed_block_rewards: Map { + mpt: recast(&source.unclaimed_block_rewards.mpt)?, + key_type: PhantomData, + }, + treasury: Map { + mpt: recast(&source.treasury.mpt)?, + key_type: PhantomData, + }, + zswap: recast(&source.zswap)?, + contract: Map { + mpt: contract_mpt.force_downcast(), + key_type: PhantomData, + }, + utxo: recast(&source.utxo)?, + replay_protection: recast(&source.replay_protection)?, + dust: recast(&source.dust)?, + })) + } +} + +// ---------- LedgerParameters v8 -> v9 ---------- + +struct LedgerParametersTl; + +impl + DirectTranslation + for LedgerParametersTl +{ + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations( + _: &ledger_v8::structure::LedgerParameters, + ) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &ledger_v8::structure::LedgerParameters, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result> { + // Base-crypto-backed fields (Duration, FixedPoint, primitives) are + // assignable directly because `midnight-base-crypto` is unified across + // v8 and v9 by workspace patches. Composite types defined in `ledger` + // (TransactionCostModel, dust parameters, etc.) are tag-stable but not + // identical types, so we go through the (de)serializer. + // + // `TransactionLimits` is the exception: v9 bumped it to + // `transaction-limits[v3]` by adding `max_contract_metadata_size`, so + // it is no longer tag-stable and is rebuilt field-by-field (its other + // fields are unified base-crypto types). + Ok(Some(ledger_v9::structure::LedgerParameters { + // `TransactionCostModel` bumped `transaction-cost-model[v4]`->`[v5]`: + // v9 drops `parallelism_factor` and adds three `FixedPoint` factors. + // The two surviving fields are tag-stable and recast through; the new + // factors get the v9 INITIAL_PARAMETERS defaults. + cost_model: ledger_v9::structure::TransactionCostModel { + runtime_cost_model: recast_base(&source.cost_model.runtime_cost_model)?, + baseline_cost: recast_base(&source.cost_model.baseline_cost)?, + // NEW IN v9 — placeholder; the production value should match the + // value chosen for the hardfork. + validation_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .validation_factor, + guaranteed_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .guaranteed_factor, + fallible_factor: ledger_v9::structure::INITIAL_PARAMETERS + .cost_model + .fallible_factor, + }, + limits: ledger_v9::structure::TransactionLimits { + transaction_byte_limit: source.limits.transaction_byte_limit, + time_to_dismiss_per_byte: source.limits.time_to_dismiss_per_byte, + min_time_to_dismiss: source.limits.min_time_to_dismiss, + block_limits: source.limits.block_limits, + block_withdrawal_minimum_multiple: source.limits.block_withdrawal_minimum_multiple, + // NEW IN v9 — placeholder; the production value should match + // the value chosen for the hardfork. + max_contract_metadata_size: ledger_v9::structure::INITIAL_PARAMETERS + .limits + .max_contract_metadata_size, + }, + dust: recast_base(&source.dust)?, + fee_prices: recast_base(&source.fee_prices)?, + global_ttl: source.global_ttl, + cost_dimension_min_ratio: source.cost_dimension_min_ratio, + price_adjustment_a_parameter: source.price_adjustment_a_parameter, + cardano_to_midnight_bridge_fee_basis_points: + source.cardano_to_midnight_bridge_fee_basis_points, + c_to_m_bridge_min_amount: source.c_to_m_bridge_min_amount, + // NEW IN v9 — placeholder; the production value should match the + // value chosen for the hardfork. + min_block_price: ledger_v9::structure::INITIAL_PARAMETERS.min_block_price, + })) + } +} + +/// Recast for tag-stable base types passed by value (cost model, limits, etc.). +/// Not the same as `recast` above which only works for `Sp`. +fn recast_base( + a: &A, +) -> io::Result { + if A::tag() != B::tag() { + return Err(io::Error::new(io::ErrorKind::Other, "tags do not match")); + } + let mut buf = Vec::new(); + a.serialize(&mut buf)?; + B::deserialize(&mut &buf[..], 0) +} + +// ---------- ContractOperation v8 -> v9 ---------- + +/// Translate a single contract operation. v9 grew `ContractOperation` from a +/// single `v2` verifier key (`contract-operation[v4]`) to `{ v2, v3, ir }` +/// (`contract-operation[v6]`). v8's only key is a zk-stdlib-v1 key +/// (`verifier-key[v6]`), which v9 keeps in its `v2` slot: that slot is backed by +/// the same `transient-crypto` 2.x crate (`transient_crypto_old`), so it is the +/// identical type and assigns directly. The new zk-stdlib-v2 `v3` key +/// (`verifier-key[v7]`, transient-crypto 3.x) and the `ir` slot have no v8 +/// equivalent and stay empty — v9 keys are *not* synthesized from v8 keys. +/// (Note `ContractOperation::new(vk, ir)` sets `v3`, not `v2`, so the struct is +/// built field-wise here.) +fn translate_contract_operation( + source: &onchain_state_v8::state::ContractOperation, +) -> onchain_state_v9::state::ContractOperation { + // `ContractOperation` is `#[non_exhaustive]`; `new` seeds `v3`/`ir`, and the + // v8 key goes into the `v2` slot field-wise. + let mut op = onchain_state_v9::state::ContractOperation::new(None, None); + op.v2 = source.v2.clone(); + op +} + +// ---------- ContractState v8 -> v9 ---------- + +struct ContractStateTl; + +impl + DirectTranslation< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + D, + > for ContractStateTl +{ + fn required_translations() -> Vec { + Vec::new() + } + fn child_translations( + _: &onchain_state_v8::state::ContractState, + ) -> Vec<(TranslationId, Sp)> { + Vec::new() + } + fn finalize( + source: &onchain_state_v8::state::ContractState, + _limit: &mut CostDuration, + _cache: &TranslationCache, + ) -> io::Result>> { + // `operations` entries (ContractOperation) changed shape, so the map + // is rebuilt entry-by-entry. The translation machinery can't walk these + // base-storable leaves nested under a contract, but a contract's + // operation set is small, so an in-place rebuild is fine. ChargedState + // and the balance map (keyed u128) are tag-stable and recast through. + let mut operations = HashMap::new(); + for entry in source.operations.iter() { + let (key, op) = &*entry; + let key_v9: onchain_state_v9::state::EntryPointBuf = key[..].into(); + operations = operations.insert(key_v9, translate_contract_operation(op)); + } + let committee_v9 = source + .maintenance_authority + .committee + .iter() + .map(|vk| { + onchain_state_v9::state::ContractMaintenanceVerifyingKey::Schnorr(vk.clone()) + }) + .collect(); + let maintenance_authority = onchain_state_v9::state::ContractMaintenanceAuthority { + committee: committee_v9, + threshold: source.maintenance_authority.threshold, + counter: source.maintenance_authority.counter, + }; + Ok(Some(onchain_state_v9::state::ContractState:: { + data: recast::< + onchain_state_v8::state::ChargedState, + onchain_state_v9::state::ChargedState, + D, + >(&Sp::new(source.data.clone()))? + .deref() + .clone(), + operations, + maintenance_authority, + balance: HashMap(Map { + mpt: recast(&source.balance.0.mpt)?, + key_type: PhantomData, + }), + })) + } +} + +// ---------- Translation table ---------- + +pub struct StateTranslationTable; + +impl TranslationTable for StateTranslationTable { + const TABLE: &[(TranslationId, &dyn TypelessTranslation)] = &[ + // Top-level + ( + TranslationId( + Cow::Borrowed("ledger-state[v13]"), + Cow::Borrowed("ledger-state[v18]"), + ), + &DirectSpTranslation::<_, _, LedgerStateTl, _>(PhantomData), + ), + // LedgerParameters + ( + TranslationId( + Cow::Borrowed("ledger-parameters[v5]"), + Cow::Borrowed("ledger-parameters[v8]"), + ), + &DirectSpTranslation::<_, _, LedgerParametersTl, _>(PhantomData), + ), + // ContractState + ( + TranslationId( + Cow::Borrowed("contract-state[v6]"), + Cow::Borrowed("contract-state[v8]"), + ), + &DirectSpTranslation::<_, _, ContractStateTl, _>(PhantomData), + ), + // `contract` MPT in LedgerState — entries are ContractState + ( + TranslationId( + Cow::Borrowed("mpt(contract-state[v6],night-annotation)"), + Cow::Borrowed("mpt(contract-state[v8],night-annotation)"), + ), + &DirectSpTranslation::< + MerklePatriciaTrie< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >, + MerklePatriciaTrie< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >, + MptTl< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + ledger_v8::annotation::NightAnn, + ledger_v9::annotation::NightAnn, + >, + _, + >(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt-node(contract-state[v6],night-annotation)"), + Cow::Borrowed("mpt-node(contract-state[v8],night-annotation)"), + ), + &DirectSpTranslation::< + merkle_patricia_trie::Node< + onchain_state_v8::state::ContractState, + D, + ledger_v8::annotation::NightAnn, + >, + merkle_patricia_trie::Node< + onchain_state_v9::state::ContractState, + D, + ledger_v9::annotation::NightAnn, + >, + MptTl< + onchain_state_v8::state::ContractState, + onchain_state_v9::state::ContractState, + ledger_v8::annotation::NightAnn, + ledger_v9::annotation::NightAnn, + >, + _, + >(PhantomData), + ), + // `bridge_receiving` MPT — entries unchanged (u128), annotation changes + // from SizeAnn to NightAnn. Needs an identity entry translation and an + // MptTl that re-annotates. + ( + TranslationId(Cow::Borrowed("u128"), Cow::Borrowed("u128")), + &DirectSpTranslation::, _>(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt(u128,size-annotation)"), + Cow::Borrowed("mpt(u128,night-annotation)"), + ), + &DirectSpTranslation::< + MerklePatriciaTrie, + MerklePatriciaTrie, + MptTl, + _, + >(PhantomData), + ), + ( + TranslationId( + Cow::Borrowed("mpt-node(u128,size-annotation)"), + Cow::Borrowed("mpt-node(u128,night-annotation)"), + ), + &DirectSpTranslation::< + merkle_patricia_trie::Node, + merkle_patricia_trie::Node, + MptTl, + _, + >(PhantomData), + ), + ]; +} diff --git a/pallets/midnight/src/lib.rs b/pallets/midnight/src/lib.rs index 0fe085103..0529af6a2 100644 --- a/pallets/midnight/src/lib.rs +++ b/pallets/midnight/src/lib.rs @@ -99,7 +99,10 @@ pub mod pallet { } } - const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + // v2: ledger v8 -> v9 state translation (see `migrations::v2`). A ledger-8 + // runtime is at on-chain version 1; upgrading to this runtime runs the + // `MigrateV1ToV2` translation. Fresh ledger-9 genesis starts at version 2. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(2); // Manually add ~1% of block weight pub const EXTRA_WEIGHT_TX_SIZE: Weight = Weight::from_parts(20_000_000_000, 0); diff --git a/pallets/midnight/src/migrations/mod.rs b/pallets/midnight/src/migrations/mod.rs index bc3f4785d..c7a82f25e 100644 --- a/pallets/midnight/src/migrations/mod.rs +++ b/pallets/midnight/src/migrations/mod.rs @@ -23,3 +23,6 @@ pub const PALLET_MIGRATIONS_ID: &[u8; 19] = b"pallet-midnight-mbm"; // See https://github.com/input-output-hk/midnight-substrate-prototype/pull/382 // for the example of such a migration. // pub mod v1; + +/// Single-block ledger v8 -> v9 state translation (storage version 1 -> 2). +pub mod v2; diff --git a/pallets/midnight/src/migrations/v2.rs b/pallets/midnight/src/migrations/v2.rs new file mode 100644 index 000000000..e04a79c86 --- /dev/null +++ b/pallets/midnight/src/migrations/v2.rs @@ -0,0 +1,98 @@ +// This file is part of midnight-node. +// Copyright (C) 2025-2026 Midnight Foundation +// SPDX-License-Identifier: Apache-2.0 +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Storage migration from v1 to v2: ledger-v8 -> ledger-v9 state translation. +//! +//! This is the on-chain half of the ledger 8 -> 9 hardfork. The pallet stores +//! only the ledger state's arena root in [`crate::StateKey`]; the `LedgerState` +//! itself lives in the ledger arena (parity-db). This migration hands the v8 +//! root to the [`migrate_state_v8_to_v9`](midnight_node_ledger::host_api::migration_8_to_9) +//! host function, which walks the v8 state, translates it into the v9 shape +//! (see [`midnight_node_ledger::state_translation_v8_to_v9`]), re-persists it, +//! and returns the new v9 root — which we write back into `StateKey`. +//! +//! It is a single-block migration: the host call translates the whole state in +//! one shot (a few milliseconds for dev/undeployed-sized state). It is wired +//! into [`crate::Migrations`](../../../runtime/src/lib.rs) via +//! [`VersionedMigration`], so it runs exactly once, when a ledger-8 runtime +//! (pallet-midnight storage version 1) upgrades to this ledger-9 runtime +//! (storage version 2). A chain whose genesis is already ledger-9 starts at +//! storage version 2 and never runs it. + +#[cfg(feature = "try-runtime")] +extern crate alloc; + +use crate::{Pallet, StateKey, pallet::Config}; +use frame_support::{ + migrations::VersionedMigration, pallet_prelude::*, traits::UncheckedOnRuntimeUpgrade, +}; +use midnight_node_ledger::types::active_ledger_bridge as LedgerApi; + +#[cfg(feature = "try-runtime")] +use alloc::vec::Vec; + +/// [`UncheckedOnRuntimeUpgrade`] implementation wrapped by [`MigrateV1ToV2`]. +pub struct InnerMigrateV1ToV2(core::marker::PhantomData); + +impl UncheckedOnRuntimeUpgrade for InnerMigrateV1ToV2 { + fn on_runtime_upgrade() -> Weight { + let state_key = StateKey::::get(); + + // The host function reads the v8 arena root, translates the referenced + // `LedgerState` to v9, re-persists it, and returns the new v9 root. + // Failure here is unrecoverable: the chain would be left pointing at a + // v8 state a ledger-9 runtime cannot read, so we abort the upgrade. + let new_state_key = LedgerApi::migrate_state_v8_to_v9(&state_key) + .expect("FATAL: ledger v8->v9 state migration failed"); + + StateKey::::put(new_state_key); + log::info!( + target: "midnight::migration", + "ledger v8->v9 state migration complete; StateKey re-pointed to v9 root" + ); + + // The heavy translation runs natively in the host call; on-chain this is + // a single `StateKey` read + write. + T::DbWeight::get().reads_writes(1, 1) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + frame_support::ensure!( + !StateKey::::get().is_empty(), + "ledger StateKey must be populated before v8->v9 migration" + ); + Ok(Vec::new()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(_state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + frame_support::ensure!( + !StateKey::::get().is_empty(), + "ledger StateKey must remain populated after v8->v9 migration" + ); + Ok(()) + } +} + +/// Translates the ledger state from v8 to v9 and bumps pallet-midnight storage +/// version 1 -> 2. Wired into the runtime `Migrations` tuple. +pub type MigrateV1ToV2 = VersionedMigration< + 1, + 2, + InnerMigrateV1ToV2, + Pallet, + ::DbWeight, +>; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 41b40d84e..595bf6fda 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -1190,7 +1190,13 @@ pub type Executive = frame_executive::Executive< /// Extrinsic type that has already been checked. pub type CheckedExtrinsic = generic::CheckedExtrinsic; /// Migrations to apply on runtime upgrade. -pub type Migrations = (pallet_throttle::migrations::v1::MigrateV0ToV1,); +pub type Migrations = ( + pallet_throttle::migrations::v1::MigrateV0ToV1, + // Ledger v8 -> v9 state translation (the ledger 8->9 hardfork). Runs once, + // when a ledger-8 runtime (pallet-midnight storage version 1) upgrades to + // this ledger-9 runtime (storage version 2). + pallet_midnight::migrations::v2::MigrateV1ToV2, +); impl frame_system::offchain::CreateTransaction for Runtime where diff --git a/util/toolkit/test-images.docker-compose.yml b/util/toolkit/test-images.docker-compose.yml index 42cdd834f..71a079559 100644 --- a/util/toolkit/test-images.docker-compose.yml +++ b/util/toolkit/test-images.docker-compose.yml @@ -15,8 +15,11 @@ services: depends_on: guard: condition: service_completed_successfully + # Ledger-8 release: the hardfork_e2e test forks from this (ledger 8, runtime + # spec_version 1_000_000, pallet-midnight storage version 1) up to the current + # ledger-9 runtime, exercising the v8->v9 state migration. midnight-node-fork-from: - image: ${FORK_FROM_NODE_IMAGE:-midnightntwrk/midnight-node:0.21.0} + image: ${FORK_FROM_NODE_IMAGE:-midnightntwrk/midnight-node:1.0.1} depends_on: guard: condition: service_completed_successfully diff --git a/util/toolkit/tests/hardfork_e2e.rs b/util/toolkit/tests/hardfork_e2e.rs index 578788807..8507a90ac 100644 --- a/util/toolkit/tests/hardfork_e2e.rs +++ b/util/toolkit/tests/hardfork_e2e.rs @@ -54,7 +54,6 @@ async fn run_cli(args: &[&str]) { } #[test_log::test(tokio::test)] -#[ignore = "Migration to Ledger v9 is not yet supported. Issue #1580."] async fn hardfork_single_tx() { // 1. Generate chain-spec from fork-from node let (old_name, old_tag) = test_image("midnight-node-fork-from"); From 17e541fc817e10a056e38467e61507bb79534d8b Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:04:33 +0100 Subject: [PATCH 02/19] test(ledger): add v8->v9 state translation table + smoke tests - table_is_closed / table_tags_match_types guard the translation table against tag drift on the node's rc.3 crate versions. - empty_state_translates_and_round_trips exercises an end-to-end v8->v9 translation and a v9 serialize round-trip. - Enable the helpers `can-panic` feature for ledger dev builds so the crate's `#[cfg(test)]` modules compile standalone. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/Cargo.toml | 5 ++ ledger/src/state_translation_v8_to_v9.rs | 110 +++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index fe10434e4..db58c54d7 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -51,6 +51,11 @@ scale-info.workspace = true [dev-dependencies] midnight-node-res = { workspace = true, features = ["test", "chain-spec"] } +# The crate's own `#[cfg(test)]` modules (api::ledger, api::transaction, and the +# state-translation tests) use `extract_tx_with_context`, gated behind the +# helpers `can-panic` feature. Enable it for test builds so the tests compile +# when the crate is tested standalone. +midnight-node-ledger-helpers = { workspace = true, features = ["can-panic", "test-utils"] } [features] default = [ diff --git a/ledger/src/state_translation_v8_to_v9.rs b/ledger/src/state_translation_v8_to_v9.rs index 9c0523d42..e4dbdd3d2 100644 --- a/ledger/src/state_translation_v8_to_v9.rs +++ b/ledger/src/state_translation_v8_to_v9.rs @@ -635,3 +635,113 @@ impl TranslationTable for StateTranslationTable { ), ]; } + +#[cfg(test)] +mod tests { + use super::*; + use storage::db::InMemoryDB; + + fn translate_to_completion( + v8: ledger_v8::structure::LedgerState, + ) -> ledger_v9::structure::LedgerState { + let tl_state = TypedTranslationState::< + ledger_v8::structure::LedgerState, + ledger_v9::structure::LedgerState, + StateTranslationTable, + InMemoryDB, + >::start(Sp::new(v8)) + .expect("Failed to start translation"); + + let cost = CostDuration::from_picoseconds(1_000_000_000_000); + let finished = tl_state.run(cost).expect("Translation failed"); + + finished + .result() + .expect("Failed to get result") + .expect("Translation did not complete") + .deref() + .clone() + } + + /// Every `TranslationId` a table entry requires must itself be in the table, + /// or translation errors at runtime the first time the entry is needed. + #[test] + fn table_is_closed() { + >::assert_closure(); + } + + /// The `TABLE` hardcodes tag string literals. If a tag on either the v8 or v9 + /// side drifts (e.g. an rc bump changes a `#[tag]`), the literal no longer + /// matches what `T::tag()` produces and the migration silently mis-dispatches. + /// Rebuild every expected ID from the node's actual crate types and compare. + #[test] + fn table_tags_match_types() { + use storage::merkle_patricia_trie::{MerklePatriciaTrie, Node}; + use storage::storable::SizeAnn; + + type V8Ann = ledger_v8::annotation::NightAnn; + type V9Ann = ledger_v9::annotation::NightAnn; + type V8Contract = onchain_state_v8::state::ContractState; + type V9Contract = onchain_state_v9::state::ContractState; + + let expected: Vec<(Cow<'static, str>, Cow<'static, str>)> = vec![ + ( + ledger_v8::structure::LedgerState::::tag(), + ledger_v9::structure::LedgerState::::tag(), + ), + ( + ledger_v8::structure::LedgerParameters::tag(), + ledger_v9::structure::LedgerParameters::tag(), + ), + (V8Contract::tag(), V9Contract::tag()), + ( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ), + ( + Node::::tag(), + Node::::tag(), + ), + (u128::tag(), u128::tag()), + ( + MerklePatriciaTrie::::tag(), + MerklePatriciaTrie::::tag(), + ), + ( + Node::::tag(), + Node::::tag(), + ), + ]; + + let actual: Vec<_> = >::TABLE + .iter() + .map(|(id, _)| (id.0.clone(), id.1.clone())) + .collect(); + + assert_eq!(actual, expected); + } + + /// End-to-end smoke test: a default v8 `LedgerState` translates to v9, + /// preserving the tag-stable pools and picking up the new v9 default + /// `min_block_price`, and survives a v9 serialize round-trip. + #[test] + fn empty_state_translates_and_round_trips() { + let v8 = ledger_v8::structure::LedgerState::::new("test-network"); + let v9 = translate_to_completion(v8.clone()); + + assert_eq!(v9.network_id, v8.network_id); + assert_eq!(v9.reserve_pool, v8.reserve_pool); + assert_eq!(v9.locked_pool, v8.locked_pool); + assert_eq!(v9.block_reward_pool, v8.block_reward_pool); + assert_eq!( + v9.parameters.min_block_price, + ledger_v9::structure::INITIAL_PARAMETERS.min_block_price, + ); + + let mut buf = Vec::new(); + serialize::tagged_serialize(&v9, &mut buf).expect("v9 serialize"); + let v9_rt: ledger_v9::structure::LedgerState = + serialize::tagged_deserialize(&mut &buf[..]).expect("v9 deserialize"); + assert_eq!(v9_rt.network_id, v9.network_id); + } +} From 8597b9a182fe040055604ba953116659f0569977 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:34:28 +0100 Subject: [PATCH 03/19] fix(node): version-aware ledger genesis seeding for the 8->9 hardfork A ledger-9 node booting on a ledger-8 chain-spec (the hardfork fork-from case) panicked at startup: the arena seeder hardcoded the ledger-9 deserializer and rejected the v8 genesis (`expected ledger-state[v18], got ledger-state[v13]`). The genesis block runs under the old WASM, so the arena must be seeded in the genesis version. - Add `genesis_matches_this_version` (per-version tag check) to common storage. - Add `init_ledger_storage_{separate,unified}` dispatchers that pick the ledger_8 vs ledger_9 seeder by the genesis `ledger-state[vN]` tag; v8 and v9 share one backend so a v8-seeded arena is what the post-migration v9 reads. - Route `custom_parity_db` through the dispatchers. - Add a dev test proving the v8-seeded root matches the fork-from chain-spec's genesisStateKey (no `Ledger`-wrapper drift vs release 1.0.1). Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/src/host_api/migration_8_to_9.rs | 39 ++++++++++++++++++++ ledger/src/lib.rs | 49 +++++++++++++++++++++++++ ledger/src/versions/common/storage.rs | 21 +++++++++++ node/src/backend/custom_parity_db.rs | 14 ++++--- 4 files changed, 118 insertions(+), 5 deletions(-) diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs index 7392b56dd..9ecc58471 100644 --- a/ledger/src/host_api/migration_8_to_9.rs +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -122,3 +122,42 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led })?; Ok(bytes) } + +#[cfg(test)] +mod tests { + use super::*; + use ledger_storage_ledger_8::db::InMemoryDB; + + /// Dev-only: verify that seeding a v8 genesis blob with *this* crate's + /// ledger_8 `Ledger` wrapper reproduces the exact arena root a ledger-8 node + /// (e.g. release 1.0.1) stored in the chain-spec as `genesisStateKey`. If the + /// wrapper serialization drifted, the seeded root wouldn't match and block + /// execution after boot would fail to find the state. Runs only when the two + /// blob paths are provided via env (extracted from a fork-from chain-spec). + #[test] + fn v8_genesis_seed_root_matches_chainspec_key() { + let (Ok(gs_path), Ok(key_path)) = + (std::env::var("HF_GENESIS_STATE"), std::env::var("HF_GENESIS_KEY")) + else { + eprintln!("skipping: set HF_GENESIS_STATE and HF_GENESIS_KEY to run"); + return; + }; + let genesis = std::fs::read(gs_path).expect("read genesis_state"); + let expected = std::fs::read(key_path).expect("read genesisStateKey"); + + let state: LedgerState8 = + midnight_serialize::tagged_deserialize(&mut &genesis[..]).expect("deserialize v8 state"); + let ledger = Ledger8::::new(state); + let mut sp = default_storage::().arena.alloc(ledger); + sp.persist(); + let mut got = Vec::new(); + tagged_serialize(&sp.as_typed_key(), &mut got).expect("serialize key"); + + assert_eq!( + got, expected, + "seeded v8 root must match the chain-spec genesisStateKey \n got={} \n exp={}", + hex::encode(&got), + hex::encode(&expected), + ); + } +} diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index da1aba91a..8057e6ba7 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -159,6 +159,55 @@ pub fn drop_all_default_storage() { ledger_9::storage::drop_default_storage_if_exists(); } +/// Seed the (separate) ledger arena from a genesis `LedgerState` blob, using the +/// deserializer that matches the blob's `ledger-state[vN]` header tag. +/// +/// A node may boot on a chain-spec produced by an older runtime — notably the +/// ledger 8->9 hardfork, where a ledger-9 node starts from a ledger-8 +/// (`ledger-state[v13]`) genesis and only upgrades to v9 later via the runtime +/// migration. Seeding must therefore match the genesis version (the genesis +/// block runs under the old WASM and expects the old-format arena root), not the +/// latest. v8 and v9 share one storage backend, so a v8-seeded arena is exactly +/// what the post-migration v9 runtime reads. Unrecognized tags fall back to the +/// latest version (`ledger_9`), preserving the prior default behaviour. +#[cfg(feature = "std")] +pub fn init_ledger_storage_separate>( + dir: P, + genesis_state: &[u8], + cache_size: usize, +) -> alloc::vec::Vec { + if ledger_8::storage::genesis_matches_this_version(genesis_state) { + ledger_8::storage::init_storage_paritydb_separate(dir, genesis_state, cache_size) + } else { + ledger_9::storage::init_storage_paritydb_separate(dir, genesis_state, cache_size) + } +} + +/// Unified-DB counterpart of [`init_ledger_storage_separate`]. +#[cfg(feature = "std")] +pub fn init_ledger_storage_unified< + D: core::ops::Deref + Default + Send + Sync + 'static, + const COLUMN_OFFSET: u8, +>( + db_instance: D, + genesis_state: &[u8], + cache_size: usize, +) -> alloc::vec::Vec { + if ledger_8::storage::genesis_matches_this_version(genesis_state) { + ledger_8::storage::init_storage_paritydb_unified::( + db_instance, + genesis_state, + cache_size, + ) + } else { + ledger_9::storage::init_storage_paritydb_unified::( + db_instance, + genesis_state, + cache_size, + ) + } +} + mod common; pub mod types { diff --git a/ledger/src/versions/common/storage.rs b/ledger/src/versions/common/storage.rs index ef39ea8d9..e57ee5e5a 100644 --- a/ledger/src/versions/common/storage.rs +++ b/ledger/src/versions/common/storage.rs @@ -74,6 +74,27 @@ impl core::fmt::Display for GetRootError { } } +/// Returns true if `genesis_state` is a tagged-serialized `LedgerState` of *this* +/// ledger version (i.e. its `midnight:ledger-state[vN]:` header tag matches this +/// version's `LedgerState::tag()`). +/// +/// Used to pick the correct version-specific seeder for the genesis arena when a +/// node boots on a chain-spec produced by an older runtime (e.g. the ledger 8->9 +/// hardfork, where a ledger-9 node starts from a ledger-8 `ledger-state[v13]` +/// genesis before the runtime upgrade migrates it to v9). +#[cfg(feature = "std")] +pub fn genesis_matches_this_version(genesis_state: &[u8]) -> bool { + use super::ledger_storage_local::DefaultDB; + let tag = as Tagged>::tag(); + // The tagged header is `midnight:{tag}:` at the very start of the payload; a + // substring search over the header region is robust to the exact global-tag + // prefix while staying anchored (tags like `ledger-state[v13]` are unique). + let head_len = genesis_state.len().min(64); + genesis_state[..head_len] + .windows(tag.len()) + .any(|w| w == tag.as_bytes()) +} + pub fn get_root(state: &[u8], network_id: Option<&str>) -> Result, GetRootError> { // Get empty state key use super::api::Ledger; diff --git a/node/src/backend/custom_parity_db.rs b/node/src/backend/custom_parity_db.rs index 5cd5ab729..ea4693256 100644 --- a/node/src/backend/custom_parity_db.rs +++ b/node/src/backend/custom_parity_db.rs @@ -109,7 +109,10 @@ pub fn open>( match storage_config.separation { StorageSeparation::Separate => { - midnight_node_ledger::ledger_9::storage::init_storage_paritydb_separate( + // Version-aware: a ledger-9 node may boot on a ledger-8 genesis during + // the 8->9 hardfork; seed the arena with the deserializer matching the + // genesis `ledger-state[vN]` tag (see `init_ledger_storage_separate`). + midnight_node_ledger::init_ledger_storage_separate( &storage_config.db_path, &storage_config.genesis_state, storage_config.cache_size, @@ -117,10 +120,11 @@ pub fn open>( Ok((OwnedDb(db), LedgerStorageDb::SeparateDb(storage_config.db_path.clone()))) }, StorageSeparation::Unified => { - midnight_node_ledger::ledger_9::storage::init_storage_paritydb_unified::< - _, - NUM_COLUMNS_POLKADOT, - >(OwnedDb(db.clone()), &storage_config.genesis_state, storage_config.cache_size); + midnight_node_ledger::init_ledger_storage_unified::<_, NUM_COLUMNS_POLKADOT>( + OwnedDb(db.clone()), + &storage_config.genesis_state, + storage_config.cache_size, + ); Ok((OwnedDb(db.clone()), LedgerStorageDb::UnifiedDb(db.clone()))) }, } From 58fe004899cc1f3b6c7e67280904706b93f60bd0 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:35:44 +0100 Subject: [PATCH 04/19] refactor(ledger): use serialize peek_tag for genesis version detection Replace the substring header scan in `genesis_matches_this_version` with `midnight_serialize::peek_tag`, matching the pattern used by `contract_operation_versioned_verifier_key`. Compares the peeked header tag against this version's `LedgerState::tag()` (no hardcoded version numbers). Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/src/versions/common/storage.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ledger/src/versions/common/storage.rs b/ledger/src/versions/common/storage.rs index e57ee5e5a..496757b52 100644 --- a/ledger/src/versions/common/storage.rs +++ b/ledger/src/versions/common/storage.rs @@ -85,14 +85,12 @@ impl core::fmt::Display for GetRootError { #[cfg(feature = "std")] pub fn genesis_matches_this_version(genesis_state: &[u8]) -> bool { use super::ledger_storage_local::DefaultDB; - let tag = as Tagged>::tag(); - // The tagged header is `midnight:{tag}:` at the very start of the payload; a - // substring search over the header region is robust to the exact global-tag - // prefix while staying anchored (tags like `ledger-state[v13]` are unique). - let head_len = genesis_state.len().min(64); - genesis_state[..head_len] - .windows(tag.len()) - .any(|w| w == tag.as_bytes()) + let expected = as Tagged>::tag(); + // `peek_tag` reads the serialized header tag without deserializing the body. + match super::midnight_serialize_local::peek_tag(&mut std::io::Cursor::new(genesis_state)) { + Ok(tag) => tag.as_str() == expected.as_ref(), + Err(_) => false, + } } pub fn get_root(state: &[u8], network_id: Option<&str>) -> Result, GetRootError> { From 7cbaaf70d743b577097a2f0980d97e05c07319d2 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:52:33 +0100 Subject: [PATCH 05/19] fix(ledger): restore ledger-8 construct_distribute_treasury host fn A ledger-9 node could not instantiate the ledger-8 runtime WASM (the hardfork fork-from case): the WASM imports `ext_ledger_8_bridge_construct_distribute_treasury_system_tx_version_1`, which was dropped from the current ledger-8 bridge (renamed to reserve/unlock-to-treasury for v9). Re-add it so the current node can execute the ledger-8 runtime across the 8->9 boundary. - ledger_8 builds `SystemTransaction::PayBlockRewardsToTreasury { amount }` (matching release 1.0.1); v7/v9 helpers are error stubs (only the ledger-8 bridge exposes the host fn). - Wire through the common Bridge and the Ledger8Bridge runtime interface. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/src/host_api/ledger_8.rs | 14 ++++++++++++++ ledger/src/versions/common/mod.rs | 6 ++++++ ledger/src/versions/system_tx/ledger_7.rs | 5 +++++ ledger/src/versions/system_tx/ledger_8.rs | 8 ++++++++ ledger/src/versions/system_tx/ledger_9.rs | 7 +++++++ 5 files changed, 40 insertions(+) diff --git a/ledger/src/host_api/ledger_8.rs b/ledger/src/host_api/ledger_8.rs index 13bf1a9c0..0e05b0d20 100644 --- a/ledger/src/host_api/ledger_8.rs +++ b/ledger/src/host_api/ledger_8.rs @@ -461,6 +461,20 @@ pub trait Ledger8Bridge { } } + /// The ledger-8 runtime imports this to pay block rewards to the treasury. + /// Retained (removed for v9) so the current node can execute the ledger-8 + /// WASM across the 8->9 hardfork boundary. + fn construct_distribute_treasury_system_tx( + &mut self, + amount: PassFatPointerAndDecode, + ) -> AllocateAndReturnByCodec, LedgerApiError>> { + if is_unified(*self) { + Bridge::::construct_distribute_treasury_system_tx(amount) + } else { + Bridge::::construct_distribute_treasury_system_tx(amount) + } + } + /// Ensures the correct ledger storage is initialized for this runtime version. /// Handles rollback: if new version's storage is initialized but we need this version's storage, /// drops new version's storage and initializes normal storage. diff --git a/ledger/src/versions/common/mod.rs b/ledger/src/versions/common/mod.rs index 270e71c5d..4ab6b467f 100644 --- a/ledger/src/versions/common/mod.rs +++ b/ledger/src/versions/common/mod.rs @@ -1157,6 +1157,12 @@ where let system_tx = super::system_tx::unlock_to_treasury_system_tx(amount)?; api.tagged_serialize(&system_tx) } + + pub fn construct_distribute_treasury_system_tx(amount: u128) -> Result, LedgerApiError> { + let api = api::new(); + let system_tx = super::system_tx::distribute_treasury_system_tx(amount)?; + api.tagged_serialize(&system_tx) + } } #[cfg(feature = "std")] diff --git a/ledger/src/versions/system_tx/ledger_7.rs b/ledger/src/versions/system_tx/ledger_7.rs index d6c5807c3..292832b0d 100644 --- a/ledger/src/versions/system_tx/ledger_7.rs +++ b/ledger/src/versions/system_tx/ledger_7.rs @@ -37,3 +37,8 @@ pub fn unlock_to_treasury_system_tx(_amount: u128) -> Result bool { false } + +/// Not applicable to ledger-7 (only the ledger-8 bridge exposes this host fn). +pub fn distribute_treasury_system_tx(_amount: u128) -> Result { + Err(LedgerApiError::HostApiError) +} diff --git a/ledger/src/versions/system_tx/ledger_8.rs b/ledger/src/versions/system_tx/ledger_8.rs index d6c5807c3..a893345eb 100644 --- a/ledger/src/versions/system_tx/ledger_8.rs +++ b/ledger/src/versions/system_tx/ledger_8.rs @@ -34,6 +34,14 @@ pub fn unlock_to_treasury_system_tx(_amount: u128) -> Result9 hardfork boundary — the ledger-8 runtime imports the corresponding +/// `construct_distribute_treasury_system_tx` host function (removed for v9). +pub fn distribute_treasury_system_tx(amount: u128) -> Result { + Ok(SystemTransaction::PayBlockRewardsToTreasury { amount }) +} + pub fn is_unlock_to_treasury_system_tx(_tx: &SystemTransaction) -> bool { false } diff --git a/ledger/src/versions/system_tx/ledger_9.rs b/ledger/src/versions/system_tx/ledger_9.rs index 6e854f4a3..99bd7a762 100644 --- a/ledger/src/versions/system_tx/ledger_9.rs +++ b/ledger/src/versions/system_tx/ledger_9.rs @@ -32,3 +32,10 @@ pub fn unlock_to_treasury_system_tx(amount: u128) -> Result bool { matches!(tx, SystemTransaction::UnlockToTreasury { .. }) } + +/// Not applicable to ledger-9: the block-rewards-to-treasury system tx was +/// removed for v9 (only the ledger-8 bridge exposes this host fn, so the ledger-8 +/// WASM can be executed across the 8->9 hardfork). +pub fn distribute_treasury_system_tx(_amount: u128) -> Result { + Err(LedgerApiError::HostApiError) +} From b00dc9705d9ff95e6462c1dbb2f30e33d86d0efc Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:16:17 +0100 Subject: [PATCH 06/19] fix(toolkit): runtime-upgrade robust to the ledger 8->9 metadata switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runtime-upgrade` used subxt's `wait_for_finalized_success()`, which eagerly decodes the apply block's events. Across the 8->9 hardfork the client's metadata follows the code swap to the new runtime, so decoding the old runtime's `System.CodeUpdated` event fails ("Can't decode field hash ..."). Wait only for finalization (`wait_for_finalized`, no event decode) and confirm the upgrade enacted by polling `state_getRuntimeVersion` for the spec_version bump — no metadata-dependent decoding across the boundary. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- util/toolkit/src/commands/runtime_upgrade.rs | 85 ++++++++++++++------ 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/util/toolkit/src/commands/runtime_upgrade.rs b/util/toolkit/src/commands/runtime_upgrade.rs index 2d145167f..cdfa39854 100644 --- a/util/toolkit/src/commands/runtime_upgrade.rs +++ b/util/toolkit/src/commands/runtime_upgrade.rs @@ -14,9 +14,13 @@ // limitations under the License. use std::str::FromStr; +use std::time::Duration; use clap::Args; -use subxt::{OnlineClient, SubstrateConfig, dynamic}; +use subxt::{ + OnlineClient, SubstrateConfig, dynamic, + rpcs::{RpcClient, rpc_params}, +}; use thiserror::Error; use crate::commands::root_call::{self, RootCallArgs}; @@ -35,6 +39,10 @@ pub enum RuntimeUpgradeError { ExtrinsicError(#[from] subxt::error::ExtrinsicError), #[error("transaction finalized error: {0}")] TransactionFinalizedError(#[from] subxt::error::TransactionFinalizedSuccessError), + #[error("transaction progress error: {0}")] + TransactionProgressError(#[from] subxt::error::TransactionProgressError), + #[error("rpc error: {0}")] + RpcError(#[from] subxt::rpcs::Error), #[error("events error: {0}")] EventsError(#[from] subxt::error::EventsError), #[error("keypair parse error: {0}")] @@ -43,6 +51,24 @@ pub enum RuntimeUpgradeError { RootCallError(Box), #[error("runtime upgrade failed: CodeUpdated event not found")] CodeUpdateNotFound, + #[error("timed out waiting for the apply_authorized_upgrade transaction to finalize")] + ApplyFinalizeTimeout, + #[error( + "runtime upgrade did not enact: spec_version stayed at {0} after applying the code" + )] + UpgradeNotEnacted(u32), +} + +/// Query the on-chain runtime spec_version via the raw `state_getRuntimeVersion` RPC. +/// +/// We avoid subxt's typed metadata here on purpose: across a runtime upgrade the +/// client's metadata switches to the new runtime, so decoding anything encoded by +/// the old runtime (e.g. the `System.CodeUpdated` event in the apply block) is +/// unreliable. The raw JSON spec_version has no such dependency. +async fn spec_version(rpc: &RpcClient) -> Result { + let version: serde_json::Value = + rpc.request("state_getRuntimeVersion", rpc_params![]).await?; + Ok(version.get("specVersion").and_then(|v| v.as_u64()).unwrap_or(0) as u32) } #[derive(Args)] @@ -78,7 +104,8 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError log::info!("Code hash: 0x{}", hex::encode(code_hash)); // Step 3: Build System::authorize_upgrade call and encode it - let api = OnlineClient::::from_insecure_url(&args.rpc_url).await?; + let rpc_client = RpcClient::from_insecure_url(&args.rpc_url).await?; + let api = OnlineClient::::from_rpc_client(rpc_client.clone()).await?; let authorize_upgrade_call = dynamic::tx("System", "authorize_upgrade", vec![dynamic::Value::from_bytes(&code_hash)]); let encoded_call = api.tx().await?.call_data(&authorize_upgrade_call)?; @@ -101,28 +128,40 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError let apply_upgrade_call = dynamic::tx("System", "apply_authorized_upgrade", vec![dynamic::Value::from_bytes(&code)]); - let apply_events = api - .tx() - .await? - .sign_and_submit_then_watch_default(&apply_upgrade_call, &signer) - .await? - .wait_for_finalized_success() - .await?; - - // Step 6: Verify CodeUpdated event - let mut success = false; - for event in apply_events.iter() { - let event = event?; - if event.pallet_name() == "System" && event.event_name() == "CodeUpdated" { - log::info!("Code update success: {:?}", event); - success = true; - break; + let pre_spec_version = spec_version(&rpc_client).await?; + log::info!("Pre-upgrade spec_version: {pre_spec_version}"); + + // Wait only for finalization — NOT `wait_for_finalized_success`, which eagerly + // decodes the block's events. `apply_authorized_upgrade` swaps the on-chain + // code, and subxt's metadata follows it to the new runtime, so decoding the + // old runtime's `System.CodeUpdated` event in that block fails. Confirm the + // upgrade succeeded by observing the spec_version bump below instead. + let submit = async { + api.tx() + .await? + .sign_and_submit_then_watch_default(&apply_upgrade_call, &signer) + .await? + .wait_for_finalized() + .await + .map_err(RuntimeUpgradeError::from) + }; + tokio::time::timeout(Duration::from_secs(120), submit) + .await + .map_err(|_| RuntimeUpgradeError::ApplyFinalizeTimeout)??; + + // Step 6: Confirm the upgrade enacted by polling for the spec_version bump. + // The new code takes effect on the block after the apply block, so poll a few + // block times. + for _ in 0..20 { + tokio::time::sleep(Duration::from_secs(3)).await; + let cur = spec_version(&rpc_client).await?; + if cur > pre_spec_version { + log::info!( + "Runtime upgrade completed successfully! spec_version {pre_spec_version} -> {cur}" + ); + return Ok(()); } } - if !success { - return Err(RuntimeUpgradeError::CodeUpdateNotFound); - } - log::info!("Runtime upgrade completed successfully!"); - Ok(()) + Err(RuntimeUpgradeError::UpgradeNotEnacted(pre_spec_version)) } From ce08d4568a2754adf232756507174aa7eed754bf Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:52:48 +0100 Subject: [PATCH 07/19] feat(toolkit): real v8->v9 fork translation at the tx-generation boundary The toolkit built a ledger-8 transaction for the post-hardfork (ledger-9) chain, which the node rejected (Deserialization(Transaction): transaction[v9]/signature[v1] vs expected transaction[v12]/signature[v2]). Its fork-aware replay never transitioned the context ledger-8 -> ledger-9. - Move StateTranslationTable into `midnight-node-ledger-helpers` (so both the runtime migration in `ledger` and the toolkit fork can use it), adding the onchain-state deps there. - `fork_context_8_to_9` now runs the real `TypedTranslationState` translation (Db8 == Db9, one shared arena) instead of the tag-reuse `old_to_new_sp`. - Wire the ledger-8 -> ledger-9 transition into `replay_blocks` (new `fork_8_to_9_if_needed`) and relax the "not supported yet" assert. - runtime-upgrade now waits for the spec bump at the FINALIZED head (toolkit fetch reads only finalized blocks), so the post-fork fetch sees the ledger-9 blocks and the replay forks to ledger 9. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- Cargo.lock | 2 + ledger/helpers/Cargo.toml | 2 + ledger/helpers/src/fork/fork_8_to_9.rs | 37 ++++++++++++++++++- ledger/helpers/src/lib.rs | 5 +++ .../src/state_translation_v8_to_v9.rs | 0 ledger/src/host_api/migration_8_to_9.rs | 2 +- ledger/src/lib.rs | 5 --- util/toolkit/src/commands/runtime_upgrade.rs | 17 ++++++--- util/toolkit/src/tx_generator/builder/mod.rs | 37 ++++++++++++++----- 9 files changed, 84 insertions(+), 23 deletions(-) rename ledger/{ => helpers}/src/state_translation_v8_to_v9.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 95cb3ade5..760de4384 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8116,6 +8116,8 @@ dependencies = [ "midnight-onchain-runtime 2.0.1", "midnight-onchain-runtime 3.1.0", "midnight-onchain-runtime 4.0.0", + "midnight-onchain-state 3.0.0", + "midnight-onchain-state 4.0.0", "midnight-serialize", "midnight-storage 1.1.1", "midnight-storage 2.0.1", diff --git a/ledger/helpers/Cargo.toml b/ledger/helpers/Cargo.toml index 76c286887..4d38158c9 100644 --- a/ledger/helpers/Cargo.toml +++ b/ledger/helpers/Cargo.toml @@ -27,11 +27,13 @@ reqwest = { workspace = true } mn-ledger-8 = { workspace = true, features = ["proving"] } onchain-runtime-ledger-8 = { workspace = true } +onchain-state-ledger-8 = { workspace = true } ledger-storage-ledger-8 = { workspace = true } zswap-ledger-8 = { workspace = true } mn-ledger-9 = { workspace = true, features = ["proving", "test-utilities"] } onchain-runtime-ledger-9 = { workspace = true } +onchain-state-ledger-9 = { workspace = true } zswap-ledger-9 = { workspace = true } coin-structure-ledger-9 = { workspace = true } transient-crypto-ledger-9 = { workspace = true } diff --git a/ledger/helpers/src/fork/fork_8_to_9.rs b/ledger/helpers/src/fork/fork_8_to_9.rs index 918cda2e9..98e972ec8 100644 --- a/ledger/helpers/src/fork/fork_8_to_9.rs +++ b/ledger/helpers/src/fork/fork_8_to_9.rs @@ -2,6 +2,10 @@ use std::collections::HashMap; use tokio::sync::Mutex as MutexTokio; +use crate::state_translation_v8_to_v9::StateTranslationTable; +use base_crypto::cost_model::CostDuration; +use ledger_storage_ledger_8::state_translation::TypedTranslationState; + type Db8 = crate::ledger_8::DefaultDB; type Db9 = crate::ledger_9::DefaultDB; @@ -67,8 +71,37 @@ pub fn fork_context_8_to_9( context8: LedgerContext8, ) -> Result, std::io::Error> { let ledger_state_8 = context8.ledger_state.lock().expect("failed to lock ledger state"); - let ledger_state: crate::ledger_9::Sp, Db8> = - old_to_new_sp(ledger_state_8.clone())?; + // Real v8->v9 state translation (NOT `old_to_new_sp`): the LedgerState tag + // changed v13->v18 and its shape changed, so a bare arena-key reuse would + // produce a v9 root the ledger-9 machinery can't read. Walk the v8 state + // through the same `StateTranslationTable` the on-chain migration uses. Db8 + // == Db9 (both `ledger_storage_ledger_8::DefaultDB`), so source and target + // share one arena and a single-`D` `TypedTranslationState` applies. + let ledger_state: crate::ledger_9::Sp, Db8> = { + let mut tl = TypedTranslationState::< + mn_ledger_8::structure::LedgerState, + mn_ledger_9::structure::LedgerState, + StateTranslationTable, + Db8, + >::start(ledger_state_8.clone())?; + // Single-shot: a generous per-step budget drains the whole state in a + // couple of iterations; the step cap is only a runaway backstop. + let budget = CostDuration::from_picoseconds(1_000_000_000_000); + let mut steps = 0usize; + loop { + steps += 1; + if steps > 100_000 { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "v8->v9 state translation did not converge", + )); + } + tl = tl.run(budget)?; + if let Some(result) = tl.result()? { + break result; + } + } + }; let mut wallets = HashMap::new(); for (k, v) in context8.wallets.lock().expect("failed to lock wallets").iter() { diff --git a/ledger/helpers/src/lib.rs b/ledger/helpers/src/lib.rs index 314cf5299..481fd0f51 100644 --- a/ledger/helpers/src/lib.rs +++ b/ledger/helpers/src/lib.rs @@ -16,6 +16,11 @@ mod utils; pub use utils::find_dependency_version; pub mod extract_tx_with_context; +/// v8 -> v9 ledger state translation table (ported from midnight-ledger PR #539). +/// Consumed by the runtime storage migration (via the `ledger` crate) and by the +/// toolkit fork boundary (`fork::fork_8_to_9`). +pub mod state_translation_v8_to_v9; + /// Strategy for ordering candidate coins/UTXOs during input selection. /// /// Defined at the crate root (not inside the version-specific `common` module) so that diff --git a/ledger/src/state_translation_v8_to_v9.rs b/ledger/helpers/src/state_translation_v8_to_v9.rs similarity index 100% rename from ledger/src/state_translation_v8_to_v9.rs rename to ledger/helpers/src/state_translation_v8_to_v9.rs diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs index 9ecc58471..08d762519 100644 --- a/ledger/src/host_api/migration_8_to_9.rs +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -33,7 +33,7 @@ use crate::ledger_8::api::Ledger as Ledger8; use crate::ledger_9::api::Ledger as Ledger9; use crate::ledger_9::types::{DeserializationError, LedgerApiError, SerializationError}; -use crate::state_translation_v8_to_v9::StateTranslationTable; +use midnight_node_ledger_helpers::state_translation_v8_to_v9::StateTranslationTable; use base_crypto::cost_model::CostDuration; use ledger_storage_ledger_8 as storage; diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index 8057e6ba7..c6c2787e6 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -27,11 +27,6 @@ pub mod json; #[cfg(feature = "std")] mod utils; -/// v8 -> v9 ledger state translation table (ported from midnight-ledger PR #539), -/// consumed by the v8->v9 storage migration host function. -#[cfg(feature = "std")] -pub mod state_translation_v8_to_v9; - pub mod host_api; #[path = "versions"] diff --git a/util/toolkit/src/commands/runtime_upgrade.rs b/util/toolkit/src/commands/runtime_upgrade.rs index cdfa39854..746d200c4 100644 --- a/util/toolkit/src/commands/runtime_upgrade.rs +++ b/util/toolkit/src/commands/runtime_upgrade.rs @@ -66,8 +66,15 @@ pub enum RuntimeUpgradeError { /// the old runtime (e.g. the `System.CodeUpdated` event in the apply block) is /// unreliable. The raw JSON spec_version has no such dependency. async fn spec_version(rpc: &RpcClient) -> Result { + // Query at the FINALIZED head, not the best block. A downstream toolkit fetch + // (`fetcher::fetch_all`) only reads up to `get_finalized_height()`, so a + // hardfork must be observed as *finalized* before we report success — + // otherwise the toolkit won't see the ledger-9 blocks yet and would build a + // transaction at the old ledger version. + let finalized_hash: serde_json::Value = + rpc.request("chain_getFinalizedHead", rpc_params![]).await?; let version: serde_json::Value = - rpc.request("state_getRuntimeVersion", rpc_params![]).await?; + rpc.request("state_getRuntimeVersion", rpc_params![finalized_hash]).await?; Ok(version.get("specVersion").and_then(|v| v.as_u64()).unwrap_or(0) as u32) } @@ -149,10 +156,10 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError .await .map_err(|_| RuntimeUpgradeError::ApplyFinalizeTimeout)??; - // Step 6: Confirm the upgrade enacted by polling for the spec_version bump. - // The new code takes effect on the block after the apply block, so poll a few - // block times. - for _ in 0..20 { + // Step 6: Confirm the upgrade enacted by polling for the spec_version bump at + // the finalized head. The new code takes effect on the block after the apply + // block, and finality lags the best block by a few blocks, so allow ~2min. + for _ in 0..40 { tokio::time::sleep(Duration::from_secs(3)).await; let cur = spec_version(&rpc_client).await?; if cur > pre_spec_version { diff --git a/util/toolkit/src/tx_generator/builder/mod.rs b/util/toolkit/src/tx_generator/builder/mod.rs index 5e113e33e..a5105200f 100644 --- a/util/toolkit/src/tx_generator/builder/mod.rs +++ b/util/toolkit/src/tx_generator/builder/mod.rs @@ -19,7 +19,7 @@ use midnight_node_ledger_helpers::fork::{ fork_aware_context::{ ForkAwareLedgerContext, apply_block_7, apply_block_8, apply_block_9, block_context_from_raw_7, block_context_from_raw_8, block_context_from_raw_9, - fork_context_7_to_8, + fork_context_7_to_8, fork_context_8_to_9, }, raw_block_data::{LedgerVersion, RawBlockData}, }; @@ -1267,6 +1267,24 @@ fn replay_blocks_9( } } +/// Fork a ledger-8 context to ledger 9 (real state translation) and replay the +/// ledger-9 blocks, if any. Returns the ledger-8 context unchanged when there are +/// no ledger-9 blocks. +fn fork_8_to_9_if_needed( + ctx8: midnight_node_ledger_helpers::ledger_8::context::LedgerContext, + l9_blocks: &[RawBlockData], + cached: &[(WalletSeed, CachedWalletState)], + schemes: &WalletSchemes, +) -> ForkAwareLedgerContext { + if l9_blocks.is_empty() { + ForkAwareLedgerContext::Ledger8(ctx8) + } else { + let ctx9 = fork_context_8_to_9(ctx8).expect("fork 8 to 9 failed"); + replay_blocks_9(&ctx9, l9_blocks, cached, schemes); + ForkAwareLedgerContext::Ledger9(ctx9) + } +} + /// Replays blocks across a potential Ledger7→Ledger8->Ledger9 fork boundaries, /// injecting cached wallets at their saved height. pub(crate) fn replay_blocks( @@ -1291,27 +1309,26 @@ pub(crate) fn replay_blocks( l8_and_l9_blocks.partition_point(|b| b.ledger_version() == LedgerVersion::Ledger8); let (l8_blocks, l9_blocks) = l8_and_l9_blocks.split_at(fork_8_to_9_idx); - assert!( - l9_blocks.is_empty() || (l7_blocks.is_empty() && l8_blocks.is_empty()), - "chain has Ledger9 blocks and eariler version blocks. This is not supported yet!" - ); - + // Replay each version's blocks in order, forking the context across the + // 7->8 and 8->9 boundaries as needed. The 8->9 fork performs a real state + // translation (see `fork_context_8_to_9`) so post-hardfork transactions are + // built at ledger 9, matching the upgraded chain. let result = match fork_ctx { ForkAwareLedgerContext::Ledger7(ctx7) => { replay_blocks_7(&ctx7, l7_blocks); - if l8_blocks.is_empty() { - assert!(cached.is_empty(), "cached wallets with no Ledger8 blocks"); + if l8_blocks.is_empty() && l9_blocks.is_empty() { + assert!(cached.is_empty(), "cached wallets with no Ledger8/9 blocks"); ForkAwareLedgerContext::Ledger7(ctx7) } else { let ctx8 = fork_context_7_to_8(ctx7).expect("fork 7 to 8 failed"); replay_blocks_8(&ctx8, l8_blocks); - ForkAwareLedgerContext::Ledger8(ctx8) + fork_8_to_9_if_needed(ctx8, l9_blocks, cached, schemes) } }, ForkAwareLedgerContext::Ledger8(ctx8) => { assert!(l7_blocks.is_empty(), "Ledger7 blocks with Ledger8 context"); replay_blocks_8(&ctx8, l8_blocks); - ForkAwareLedgerContext::Ledger8(ctx8) + fork_8_to_9_if_needed(ctx8, l9_blocks, cached, schemes) }, ForkAwareLedgerContext::Ledger9(ctx9) => { assert!(l7_blocks.is_empty(), "Ledger7 blocks with Ledger9 context"); From 0ee9e299ca54fbf87ff07a50742cc4c21d955699 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:10:06 +0100 Subject: [PATCH 08/19] chore(toolkit): TEMP hardfork_debug log for replay_blocks partition Temporary diagnostic to see the l7/l8/l9 block partition + initial context version during the post-fork tx build. Remove once the fork boundary is fixed. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- util/toolkit/src/tx_generator/builder/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/util/toolkit/src/tx_generator/builder/mod.rs b/util/toolkit/src/tx_generator/builder/mod.rs index a5105200f..31ed9c9c4 100644 --- a/util/toolkit/src/tx_generator/builder/mod.rs +++ b/util/toolkit/src/tx_generator/builder/mod.rs @@ -1309,6 +1309,17 @@ pub(crate) fn replay_blocks( l8_and_l9_blocks.partition_point(|b| b.ledger_version() == LedgerVersion::Ledger8); let (l8_blocks, l9_blocks) = l8_and_l9_blocks.split_at(fork_8_to_9_idx); + log::info!( + target: "hardfork_debug", + "replay_blocks: total={} l7={} l8={} l9={} initial_ctx={:?} last_block_ver={:?}", + blocks.len(), + l7_blocks.len(), + l8_blocks.len(), + l9_blocks.len(), + fork_ctx.version(), + blocks.last().map(|b| b.ledger_version()), + ); + // Replay each version's blocks in order, forking the context across the // 7->8 and 8->9 boundaries as needed. The 8->9 fork performs a real state // translation (see `fork_context_8_to_9`) so post-hardfork transactions are From 43808b22340feee4099f302ea11b16af452c4f54 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:24:36 +0100 Subject: [PATCH 09/19] fix(toolkit): runtime-upgrade waits for the new runtime to EXECUTE, not just apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit state_getRuntimeVersion(finalized) reports the *stored* code, which flips to the new runtime at the apply_authorized_upgrade block — but that block still *executes* under the old runtime (its MNSV digest, which the fetcher uses to classify ledger version, is the old spec). The first block to run the new runtime is apply+1. The prior poll returned at the apply block, so a downstream fetch (bounded by finalized height) reached the apply block (still classified ledger-8) but not apply+1, and the toolkit built a ledger-8 tx post-fork. Track the finalized height where the stored spec first bumps, then wait for the finalized height to advance past it (apply+1 finalized) before reporting success. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- util/toolkit/src/commands/runtime_upgrade.rs | 70 ++++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/util/toolkit/src/commands/runtime_upgrade.rs b/util/toolkit/src/commands/runtime_upgrade.rs index 746d200c4..935b3c2a2 100644 --- a/util/toolkit/src/commands/runtime_upgrade.rs +++ b/util/toolkit/src/commands/runtime_upgrade.rs @@ -65,17 +65,29 @@ pub enum RuntimeUpgradeError { /// client's metadata switches to the new runtime, so decoding anything encoded by /// the old runtime (e.g. the `System.CodeUpdated` event in the apply block) is /// unreliable. The raw JSON spec_version has no such dependency. -async fn spec_version(rpc: &RpcClient) -> Result { - // Query at the FINALIZED head, not the best block. A downstream toolkit fetch - // (`fetcher::fetch_all`) only reads up to `get_finalized_height()`, so a - // hardfork must be observed as *finalized* before we report success — - // otherwise the toolkit won't see the ledger-9 blocks yet and would build a - // transaction at the old ledger version. - let finalized_hash: serde_json::Value = - rpc.request("chain_getFinalizedHead", rpc_params![]).await?; +/// (finalized_height, spec_version at the finalized head). +/// +/// We track both because `state_getRuntimeVersion(finalized)` reports the code +/// *stored at* that block — which flips to the new runtime already at the +/// `apply_authorized_upgrade` block, even though that block *executed* under the +/// old runtime (its `MNSV` digest — how the toolkit fetcher classifies a block's +/// ledger version — is still the old spec). The first block that actually +/// executes the new runtime is `apply + 1`. So "spec flipped at the finalized +/// head" is one block early; callers must additionally wait for the finalized +/// height to advance past that point before the fetcher will see a ledger-9 block. +async fn finalized_state(rpc: &RpcClient) -> Result<(u64, u32), RuntimeUpgradeError> { + let hash: serde_json::Value = rpc.request("chain_getFinalizedHead", rpc_params![]).await?; + let header: serde_json::Value = + rpc.request("chain_getHeader", rpc_params![hash.clone()]).await?; let version: serde_json::Value = - rpc.request("state_getRuntimeVersion", rpc_params![finalized_hash]).await?; - Ok(version.get("specVersion").and_then(|v| v.as_u64()).unwrap_or(0) as u32) + rpc.request("state_getRuntimeVersion", rpc_params![hash]).await?; + let height = header + .get("number") + .and_then(|n| n.as_str()) + .and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + .unwrap_or(0); + let spec = version.get("specVersion").and_then(|v| v.as_u64()).unwrap_or(0) as u32; + Ok((height, spec)) } #[derive(Args)] @@ -135,8 +147,8 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError let apply_upgrade_call = dynamic::tx("System", "apply_authorized_upgrade", vec![dynamic::Value::from_bytes(&code)]); - let pre_spec_version = spec_version(&rpc_client).await?; - log::info!("Pre-upgrade spec_version: {pre_spec_version}"); + let (_, pre_spec_version) = finalized_state(&rpc_client).await?; + log::info!("Pre-upgrade spec_version (finalized): {pre_spec_version}"); // Wait only for finalization — NOT `wait_for_finalized_success`, which eagerly // decodes the block's events. `apply_authorized_upgrade` swaps the on-chain @@ -156,17 +168,31 @@ pub async fn execute(args: RuntimeUpgradeArgs) -> Result<(), RuntimeUpgradeError .await .map_err(|_| RuntimeUpgradeError::ApplyFinalizeTimeout)??; - // Step 6: Confirm the upgrade enacted by polling for the spec_version bump at - // the finalized head. The new code takes effect on the block after the apply - // block, and finality lags the best block by a few blocks, so allow ~2min. - for _ in 0..40 { + // Step 6: Confirm the upgrade is not just applied but *executing* at a + // finalized block. `state_getRuntimeVersion(finalized)` reports the stored + // code, which flips to the new runtime already at the apply block — but that + // block's `MNSV` execution-version digest (how the fetcher classifies it) is + // still the old spec. The first block that runs the new runtime is apply+1. + // So: note the finalized height where the stored spec first exceeds pre, then + // wait for the finalized height to advance past it (apply+1 finalized). Only + // then will a downstream `fetch` see a ledger-9-classified block. + let mut flip_height: Option = None; + for _ in 0..60 { tokio::time::sleep(Duration::from_secs(3)).await; - let cur = spec_version(&rpc_client).await?; - if cur > pre_spec_version { - log::info!( - "Runtime upgrade completed successfully! spec_version {pre_spec_version} -> {cur}" - ); - return Ok(()); + let (height, spec) = finalized_state(&rpc_client).await?; + if spec > pre_spec_version { + match flip_height { + None => flip_height = Some(height), + Some(h0) if height > h0 => { + log::info!( + "Runtime upgrade completed successfully! spec_version {pre_spec_version} -> {spec}; \ + new runtime executing since finalized #{}, now finalized #{height}", + h0 + 1, + ); + return Ok(()); + }, + _ => {}, + } } } From af7ed4f814bdbd7d91c3021e5cd91cf3e3867e07 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:36:29 +0100 Subject: [PATCH 10/19] chore(toolkit): remove TEMP hardfork_debug replay_blocks log The v8->v9 fork boundary is fixed and hardfork_e2e passes; drop the diagnostic. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- util/toolkit/src/tx_generator/builder/mod.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/util/toolkit/src/tx_generator/builder/mod.rs b/util/toolkit/src/tx_generator/builder/mod.rs index 31ed9c9c4..a5105200f 100644 --- a/util/toolkit/src/tx_generator/builder/mod.rs +++ b/util/toolkit/src/tx_generator/builder/mod.rs @@ -1309,17 +1309,6 @@ pub(crate) fn replay_blocks( l8_and_l9_blocks.partition_point(|b| b.ledger_version() == LedgerVersion::Ledger8); let (l8_blocks, l9_blocks) = l8_and_l9_blocks.split_at(fork_8_to_9_idx); - log::info!( - target: "hardfork_debug", - "replay_blocks: total={} l7={} l8={} l9={} initial_ctx={:?} last_block_ver={:?}", - blocks.len(), - l7_blocks.len(), - l8_blocks.len(), - l9_blocks.len(), - fork_ctx.version(), - blocks.last().map(|b| b.ledger_version()), - ); - // Replay each version's blocks in order, forking the context across the // 7->8 and 8->9 boundaries as needed. The 8->9 fork performs a real state // translation (see `fork_context_8_to_9`) so post-hardfork transactions are From 4265e2b4816d6d40837c9acd43d41bea1f6c9ec9 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:55:19 +0100 Subject: [PATCH 11/19] docs: add change files for the ledger 8->9 hardfork migration Assisted-by: Claude:claude-sonnet-5 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- .../ledger-8-to-9-hardfork-migration.md | 21 +++++++++++++++++++ .../hardfork-fork-aware-tx-generation.md | 18 ++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 changes/runtime/changed/ledger-8-to-9-hardfork-migration.md create mode 100644 changes/toolkit/changed/hardfork-fork-aware-tx-generation.md diff --git a/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md new file mode 100644 index 000000000..6bb3b15ab --- /dev/null +++ b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md @@ -0,0 +1,21 @@ +#node #runtime #ledger + +# On-chain ledger 8->9 hardfork state migration + +Lets a ledger-8 chain (e.g. `1.0.1`) runtime-upgrade in place to the current +ledger-9 runtime. A new host fn, `migrate_state_v8_to_v9`, runs the +`StateTranslationTable` (ported from `midnight-ledger` PR #539) to translate +the on-chain `LedgerState` from v13 to v18. It's wired in as +`pallet_midnight::migrations::v2::MigrateV1ToV2`, a `VersionedMigration<1,2,..>` +that fires once when a ledger-8 chain (pallet-midnight storage version 1) +upgrades to this runtime (storage version 2); a fresh ledger-9 genesis starts +at version 2 and skips it. + +Also includes two fixes needed to support both ledger-8 and ledger-9 chains: +version-aware genesis seeding (detected via `serialize::peek_tag` instead of +hardcoding the v9 deserializer), and restoring the ledger-8 +`construct_distribute_treasury_system_tx` host fn, which the `1.0.1` WASM +still imports. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1925 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1580 diff --git a/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md b/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md new file mode 100644 index 000000000..9fc711c0e --- /dev/null +++ b/changes/toolkit/changed/hardfork-fork-aware-tx-generation.md @@ -0,0 +1,18 @@ +#toolkit #ledger9 + +# Fork-aware transaction generation across the ledger 8->9 hardfork + +`replay_blocks` now detects the v8->v9 fork boundary and runs the same +`StateTranslationTable` translation used by the on-chain migration +(`fork_context_8_to_9` / `fork_8_to_9_if_needed`), so transactions generated +after the fork are built against the correctly-translated ledger-9 context +instead of a stale ledger-8 one. + +The `runtime-upgrade` command now waits for the new runtime to actually +*execute* at a finalized block, not just be applied/stored. The stored spec +version flips at the apply block, but that block still executes under the +old runtime, so polling only the stored spec left transaction generation +reading a block short of any ledger-9-classified block. + +PR: https://github.com/midnightntwrk/midnight-node/pull/1925 +Issue: https://github.com/midnightntwrk/midnight-node/issues/1580 From e30976cffad6b8259487c065c14c8b8f519ffccc Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:42:02 +0100 Subject: [PATCH 12/19] chore: cargo fmt Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- .../helpers/src/state_translation_v8_to_v9.rs | 101 ++++++------------ ledger/src/host_api/migration_8_to_9.rs | 28 ++--- ledger/src/versions/common/mod.rs | 4 +- util/toolkit/src/commands/runtime_upgrade.rs | 4 +- 4 files changed, 50 insertions(+), 87 deletions(-) diff --git a/ledger/helpers/src/state_translation_v8_to_v9.rs b/ledger/helpers/src/state_translation_v8_to_v9.rs index e4dbdd3d2..545275ed3 100644 --- a/ledger/helpers/src/state_translation_v8_to_v9.rs +++ b/ledger/helpers/src/state_translation_v8_to_v9.rs @@ -106,9 +106,7 @@ impl< cache: &TranslationCache, ) -> io::Result>> { let tls = Self::child_translations(source); - Ok(Some(MerklePatriciaTrie(try_resopt!( - cache.resolve(&tls[0].0, tls[0].1.as_child()) - )))) + Ok(Some(MerklePatriciaTrie(try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child()))))) } } @@ -141,16 +139,15 @@ impl< let self_tl = tls[1].clone(); match source { merkle_patricia_trie::Node::Empty => vec![], - merkle_patricia_trie::Node::Branch { children, .. } => children - .iter() - .map(|child| (self_tl.clone(), child.upcast())) - .collect(), + merkle_patricia_trie::Node::Branch { children, .. } => { + children.iter().map(|child| (self_tl.clone(), child.upcast())).collect() + }, merkle_patricia_trie::Node::Extension { child, .. } => { vec![(self_tl, child.upcast())] - } + }, merkle_patricia_trie::Node::MidBranchLeaf { value, child, .. } => { vec![(entry_tl, value.upcast()), (self_tl, child.upcast())] - } + }, merkle_patricia_trie::Node::Leaf { value, .. } => vec![(entry_tl, value.upcast())], } } @@ -168,19 +165,12 @@ impl< for (child, new_child) in tls.iter().zip(new_children.iter_mut()) { *new_child = try_resopt!(cache.resolve(&child.0, child.1.as_child())); } - let ann = new_children - .iter() - .fold(AnnB::empty(), |acc, x| { - acc.append(&merkle_patricia_trie::Node::::ann(x)) - }); - merkle_patricia_trie::Node::Branch { - ann, - children: Box::new(new_children), - } - } - merkle_patricia_trie::Node::Extension { - compressed_path, .. - } => { + let ann = new_children.iter().fold(AnnB::empty(), |acc, x| { + acc.append(&merkle_patricia_trie::Node::::ann(x)) + }); + merkle_patricia_trie::Node::Branch { ann, children: Box::new(new_children) } + }, + merkle_patricia_trie::Node::Extension { compressed_path, .. } => { let child: Sp, D> = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); let ann = merkle_patricia_trie::Node::::ann(&child); @@ -189,12 +179,12 @@ impl< compressed_path: compressed_path.clone(), child, } - } + }, merkle_patricia_trie::Node::Leaf { .. } => { let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); let ann = AnnB::from_value(&value); merkle_patricia_trie::Node::Leaf { ann, value } - } + }, merkle_patricia_trie::Node::MidBranchLeaf { .. } => { let value = try_resopt!(cache.resolve(&tls[0].0, tls[0].1.as_child())); let child: Sp, D> = @@ -202,7 +192,7 @@ impl< let ann = AnnB::from_value(&value) .append(&merkle_patricia_trie::Node::::ann(&child)); merkle_patricia_trie::Node::MidBranchLeaf { ann, value, child } - } + }, })) } } @@ -272,11 +262,7 @@ impl for LedgerStateTl { fn required_translations() -> Vec { - vec![ - Ids::parameters::(), - Ids::bridge_receiving_mpt::(), - Ids::contract_mpt::(), - ] + vec![Ids::parameters::(), Ids::bridge_receiving_mpt::(), Ids::contract_mpt::()] } fn child_translations( @@ -284,10 +270,7 @@ impl ) -> Vec<(TranslationId, Sp)> { vec![ (Ids::parameters::(), source.parameters.upcast()), - ( - Ids::bridge_receiving_mpt::(), - source.bridge_receiving.mpt.upcast(), - ), + (Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.upcast()), (Ids::contract_mpt::(), source.contract.mpt.upcast()), ] } @@ -316,25 +299,16 @@ impl network_id: source.network_id.clone(), parameters: parameters.force_downcast(), locked_pool: source.locked_pool, - bridge_receiving: Map { - mpt: bridge_recv_mpt.force_downcast(), - key_type: PhantomData, - }, + bridge_receiving: Map { mpt: bridge_recv_mpt.force_downcast(), key_type: PhantomData }, reserve_pool: source.reserve_pool, block_reward_pool: source.block_reward_pool, unclaimed_block_rewards: Map { mpt: recast(&source.unclaimed_block_rewards.mpt)?, key_type: PhantomData, }, - treasury: Map { - mpt: recast(&source.treasury.mpt)?, - key_type: PhantomData, - }, + treasury: Map { mpt: recast(&source.treasury.mpt)?, key_type: PhantomData }, zswap: recast(&source.zswap)?, - contract: Map { - mpt: contract_mpt.force_downcast(), - key_type: PhantomData, - }, + contract: Map { mpt: contract_mpt.force_downcast(), key_type: PhantomData }, utxo: recast(&source.utxo)?, replay_protection: recast(&source.replay_protection)?, dust: recast(&source.dust)?, @@ -347,8 +321,11 @@ impl struct LedgerParametersTl; impl - DirectTranslation - for LedgerParametersTl + DirectTranslation< + ledger_v8::structure::LedgerParameters, + ledger_v9::structure::LedgerParameters, + D, + > for LedgerParametersTl { fn required_translations() -> Vec { Vec::new() @@ -410,8 +387,8 @@ impl global_ttl: source.global_ttl, cost_dimension_min_ratio: source.cost_dimension_min_ratio, price_adjustment_a_parameter: source.price_adjustment_a_parameter, - cardano_to_midnight_bridge_fee_basis_points: - source.cardano_to_midnight_bridge_fee_basis_points, + cardano_to_midnight_bridge_fee_basis_points: source + .cardano_to_midnight_bridge_fee_basis_points, c_to_m_bridge_min_amount: source.c_to_m_bridge_min_amount, // NEW IN v9 — placeholder; the production value should match the // value chosen for the hardfork. @@ -494,9 +471,7 @@ impl .maintenance_authority .committee .iter() - .map(|vk| { - onchain_state_v9::state::ContractMaintenanceVerifyingKey::Schnorr(vk.clone()) - }) + .map(|vk| onchain_state_v9::state::ContractMaintenanceVerifyingKey::Schnorr(vk.clone())) .collect(); let maintenance_authority = onchain_state_v9::state::ContractMaintenanceAuthority { committee: committee_v9, @@ -513,10 +488,7 @@ impl .clone(), operations, maintenance_authority, - balance: HashMap(Map { - mpt: recast(&source.balance.0.mpt)?, - key_type: PhantomData, - }), + balance: HashMap(Map { mpt: recast(&source.balance.0.mpt)?, key_type: PhantomData }), })) } } @@ -529,10 +501,7 @@ impl TranslationTable for StateTranslationTable { const TABLE: &[(TranslationId, &dyn TypelessTranslation)] = &[ // Top-level ( - TranslationId( - Cow::Borrowed("ledger-state[v13]"), - Cow::Borrowed("ledger-state[v18]"), - ), + TranslationId(Cow::Borrowed("ledger-state[v13]"), Cow::Borrowed("ledger-state[v18]")), &DirectSpTranslation::<_, _, LedgerStateTl, _>(PhantomData), ), // LedgerParameters @@ -545,10 +514,7 @@ impl TranslationTable for StateTranslationTable { ), // ContractState ( - TranslationId( - Cow::Borrowed("contract-state[v6]"), - Cow::Borrowed("contract-state[v8]"), - ), + TranslationId(Cow::Borrowed("contract-state[v6]"), Cow::Borrowed("contract-state[v8]")), &DirectSpTranslation::<_, _, ContractStateTl, _>(PhantomData), ), // `contract` MPT in LedgerState — entries are ContractState @@ -707,10 +673,7 @@ mod tests { MerklePatriciaTrie::::tag(), MerklePatriciaTrie::::tag(), ), - ( - Node::::tag(), - Node::::tag(), - ), + (Node::::tag(), Node::::tag()), ]; let actual: Vec<_> = >::TABLE diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs index 08d762519..dd3a57a48 100644 --- a/ledger/src/host_api/migration_8_to_9.rs +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -63,8 +63,8 @@ const MAX_STEPS: usize = 100_000; /// the new v9 arena root to store back into `StateKey`. pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, LedgerApiError> { // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. - let key8: TypedArenaKey, D::Hasher> = - tagged_deserialize(&mut &state_key_v8[..]).map_err(|e| { + let key8: TypedArenaKey, D::Hasher> = tagged_deserialize(&mut &state_key_v8[..]) + .map_err(|e| { log::error!(target: LOG_TARGET, "failed to deserialize v8 state key: {e:?}"); LedgerApiError::Deserialization(DeserializationError::TypedArenaKey) })?; @@ -75,16 +75,14 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led // 2. Run the state translation table over the inner v8 `LedgerState`. let input: Sp, D> = Sp::new(ledger8.state.clone()); - let mut tl = TypedTranslationState::< - LedgerState8, - LedgerState9, - StateTranslationTable, - D, - >::start(input) - .map_err(|e| { - log::error!(target: LOG_TARGET, "failed to start v8->v9 translation: {e:?}"); - LedgerApiError::HostApiError - })?; + let mut tl = + TypedTranslationState::, LedgerState9, StateTranslationTable, D>::start( + input, + ) + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to start v8->v9 translation: {e:?}"); + LedgerApiError::HostApiError + })?; let run_budget = CostDuration::from_picoseconds(RUN_BUDGET_PS); let mut steps = 0usize; @@ -146,7 +144,8 @@ mod tests { let expected = std::fs::read(key_path).expect("read genesisStateKey"); let state: LedgerState8 = - midnight_serialize::tagged_deserialize(&mut &genesis[..]).expect("deserialize v8 state"); + midnight_serialize::tagged_deserialize(&mut &genesis[..]) + .expect("deserialize v8 state"); let ledger = Ledger8::::new(state); let mut sp = default_storage::().arena.alloc(ledger); sp.persist(); @@ -154,7 +153,8 @@ mod tests { tagged_serialize(&sp.as_typed_key(), &mut got).expect("serialize key"); assert_eq!( - got, expected, + got, + expected, "seeded v8 root must match the chain-spec genesisStateKey \n got={} \n exp={}", hex::encode(&got), hex::encode(&expected), diff --git a/ledger/src/versions/common/mod.rs b/ledger/src/versions/common/mod.rs index 4ab6b467f..0c9b72588 100644 --- a/ledger/src/versions/common/mod.rs +++ b/ledger/src/versions/common/mod.rs @@ -1158,7 +1158,9 @@ where api.tagged_serialize(&system_tx) } - pub fn construct_distribute_treasury_system_tx(amount: u128) -> Result, LedgerApiError> { + pub fn construct_distribute_treasury_system_tx( + amount: u128, + ) -> Result, LedgerApiError> { let api = api::new(); let system_tx = super::system_tx::distribute_treasury_system_tx(amount)?; api.tagged_serialize(&system_tx) diff --git a/util/toolkit/src/commands/runtime_upgrade.rs b/util/toolkit/src/commands/runtime_upgrade.rs index 935b3c2a2..367b6daf2 100644 --- a/util/toolkit/src/commands/runtime_upgrade.rs +++ b/util/toolkit/src/commands/runtime_upgrade.rs @@ -53,9 +53,7 @@ pub enum RuntimeUpgradeError { CodeUpdateNotFound, #[error("timed out waiting for the apply_authorized_upgrade transaction to finalize")] ApplyFinalizeTimeout, - #[error( - "runtime upgrade did not enact: spec_version stayed at {0} after applying the code" - )] + #[error("runtime upgrade did not enact: spec_version stayed at {0} after applying the code")] UpgradeNotEnacted(u32), } From 08671ba18d162f87d0213f112ca363e7baafd5f5 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:25:02 +0100 Subject: [PATCH 13/19] chore: fix clippy errors Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/helpers/src/fork/fork_8_to_9.rs | 5 +---- ledger/helpers/src/state_translation_v8_to_v9.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ledger/helpers/src/fork/fork_8_to_9.rs b/ledger/helpers/src/fork/fork_8_to_9.rs index 98e972ec8..0feb2aa03 100644 --- a/ledger/helpers/src/fork/fork_8_to_9.rs +++ b/ledger/helpers/src/fork/fork_8_to_9.rs @@ -91,10 +91,7 @@ pub fn fork_context_8_to_9( loop { steps += 1; if steps > 100_000 { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "v8->v9 state translation did not converge", - )); + return Err(std::io::Error::other("v8->v9 state translation did not converge")); } tl = tl.run(budget)?; if let Some(result) = tl.result()? { diff --git a/ledger/helpers/src/state_translation_v8_to_v9.rs b/ledger/helpers/src/state_translation_v8_to_v9.rs index 545275ed3..4106b7819 100644 --- a/ledger/helpers/src/state_translation_v8_to_v9.rs +++ b/ledger/helpers/src/state_translation_v8_to_v9.rs @@ -69,7 +69,7 @@ fn recast + Tagged, B: Storable + Tagged, D: DB>( a: &Sp, ) -> io::Result> { if A::tag() != B::tag() { - return io::Result::Err(io::Error::new(io::ErrorKind::Other, "tags do not match")); + return io::Result::Err(io::Error::other("tags do not match")); } default_storage::().get_lazy(&a.as_child().into()) } @@ -245,7 +245,7 @@ impl Ids { ) } - fn parameters() -> TranslationId { + fn parameters() -> TranslationId { TranslationId( ledger_v8::structure::LedgerParameters::tag(), ledger_v9::structure::LedgerParameters::tag(), @@ -262,14 +262,14 @@ impl for LedgerStateTl { fn required_translations() -> Vec { - vec![Ids::parameters::(), Ids::bridge_receiving_mpt::(), Ids::contract_mpt::()] + vec![Ids::parameters(), Ids::bridge_receiving_mpt::(), Ids::contract_mpt::()] } fn child_translations( source: &ledger_v8::structure::LedgerState, ) -> Vec<(TranslationId, Sp)> { vec![ - (Ids::parameters::(), source.parameters.upcast()), + (Ids::parameters(), source.parameters.upcast()), (Ids::bridge_receiving_mpt::(), source.bridge_receiving.mpt.upcast()), (Ids::contract_mpt::(), source.contract.mpt.upcast()), ] @@ -280,7 +280,7 @@ impl _limit: &mut CostDuration, cache: &TranslationCache, ) -> io::Result>> { - let Some(parameters) = cache.lookup(&Ids::parameters::(), source.parameters.as_child()) + let Some(parameters) = cache.lookup(&Ids::parameters(), source.parameters.as_child()) else { return Ok(None); }; @@ -403,7 +403,7 @@ fn recast_base io::Result { if A::tag() != B::tag() { - return Err(io::Error::new(io::ErrorKind::Other, "tags do not match")); + return Err(io::Error::other("tags do not match")); } let mut buf = Vec::new(); a.serialize(&mut buf)?; From e8c1427d5aec1d580390481703495aad6740d056 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:40:46 +0100 Subject: [PATCH 14/19] chore: npm audit fix (toolkit-js) Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- util/toolkit-js/package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/util/toolkit-js/package-lock.json b/util/toolkit-js/package-lock.json index ef0d5abec..73bf83929 100644 --- a/util/toolkit-js/package-lock.json +++ b/util/toolkit-js/package-lock.json @@ -3271,9 +3271,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "dev": true, "funding": [ { @@ -3421,9 +3421,9 @@ } }, "node_modules/postcss": { - "version": "8.5.16", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", - "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", "dev": true, "funding": [ { @@ -3441,7 +3441,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, From 679a1a512e8fcc8de4f886c90bfb75c939d4045d Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:56:28 +0100 Subject: [PATCH 15/19] feat: add perf prints to track migration time elapsed Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/helpers/src/fork/fork_8_to_9.rs | 1 + ledger/src/host_api/migration_8_to_9.rs | 19 ++++++++++++++++++- util/toolkit/src/tx_generator/builder/mod.rs | 6 ++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/ledger/helpers/src/fork/fork_8_to_9.rs b/ledger/helpers/src/fork/fork_8_to_9.rs index 0feb2aa03..fac03ec49 100644 --- a/ledger/helpers/src/fork/fork_8_to_9.rs +++ b/ledger/helpers/src/fork/fork_8_to_9.rs @@ -86,6 +86,7 @@ pub fn fork_context_8_to_9( >::start(ledger_state_8.clone())?; // Single-shot: a generous per-step budget drains the whole state in a // couple of iterations; the step cap is only a runaway backstop. + // 1_000_000_000_000 pico-seconds == 1 second let budget = CostDuration::from_picoseconds(1_000_000_000_000); let mut steps = 0usize; loop { diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs index dd3a57a48..8ceaead7a 100644 --- a/ledger/src/host_api/migration_8_to_9.rs +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -62,7 +62,10 @@ const MAX_STEPS: usize = 100_000; /// the pallet's `StateKey` bytes) into a v9 ledger state, persist it, and return /// the new v9 arena root to store back into `StateKey`. pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, LedgerApiError> { + let t_total = std::time::Instant::now(); + // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. + let t_load = std::time::Instant::now(); let key8: TypedArenaKey, D::Hasher> = tagged_deserialize(&mut &state_key_v8[..]) .map_err(|e| { log::error!(target: LOG_TARGET, "failed to deserialize v8 state key: {e:?}"); @@ -72,8 +75,10 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led log::error!(target: LOG_TARGET, "failed to load v8 ledger from arena: {e:?}"); LedgerApiError::NoLedgerState })?; + log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 load took {:?}", t_load.elapsed()); // 2. Run the state translation table over the inner v8 `LedgerState`. + let t_translate = std::time::Instant::now(); let input: Sp, D> = Sp::new(ledger8.state.clone()); let mut tl = TypedTranslationState::, LedgerState9, StateTranslationTable, D>::start( @@ -103,14 +108,24 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led break result; } }; - log::info!(target: LOG_TARGET, "v8->v9 ledger state translation complete in {steps} step(s)"); + log::info!( + target: LOG_TARGET, + "v8->v9 ledger state translation complete in {steps} step(s), {:?}", + t_translate.elapsed() + ); // 3. Wrap the translated state in the v9 ledger wrapper, persist it, and // flush the arena so the new root is durable before the pallet stores it. + let t_persist = std::time::Instant::now(); let ledger9 = Ledger9::new((*state9).clone()); let mut sp9: Sp, D> = default_storage::().arena.alloc(ledger9); sp9.persist(); default_storage::().with_backend(|backend| backend.flush_all_changes_to_db()); + log::debug!( + target: LOG_TARGET, + "[perf] migrate_state_v8_to_v9 persist+flush took {:?}", + t_persist.elapsed() + ); // 4. Serialize the new v9 arena root for the pallet to store in `StateKey`. let mut bytes = Vec::new(); @@ -118,6 +133,8 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led log::error!(target: LOG_TARGET, "failed to serialize v9 state key: {e:?}"); LedgerApiError::Serialization(SerializationError::TypedArenaKey) })?; + + log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 took {:?}", t_total.elapsed()); Ok(bytes) } diff --git a/util/toolkit/src/tx_generator/builder/mod.rs b/util/toolkit/src/tx_generator/builder/mod.rs index f1bef811c..ade6566f7 100644 --- a/util/toolkit/src/tx_generator/builder/mod.rs +++ b/util/toolkit/src/tx_generator/builder/mod.rs @@ -1346,7 +1346,8 @@ fn fork_8_to_9_if_needed( if l9_blocks.is_empty() { ForkAwareLedgerContext::Ledger8(ctx8) } else { - let ctx9 = fork_context_8_to_9(ctx8).expect("fork 8 to 9 failed"); + let ctx9 = + timed!("fork_context_8_to_9", fork_context_8_to_9(ctx8)).expect("fork 8 to 9 failed"); replay_blocks_9(&ctx9, l9_blocks, cached, schemes); ForkAwareLedgerContext::Ledger9(ctx9) } @@ -1387,7 +1388,8 @@ pub(crate) fn replay_blocks( assert!(cached.is_empty(), "cached wallets with no Ledger8/9 blocks"); ForkAwareLedgerContext::Ledger7(ctx7) } else { - let ctx8 = fork_context_7_to_8(ctx7).expect("fork 7 to 8 failed"); + let ctx8 = timed!("fork_context_7_to_8", fork_context_7_to_8(ctx7)) + .expect("fork 7 to 8 failed"); replay_blocks_8(&ctx8, l8_blocks); fork_8_to_9_if_needed(ctx8, l9_blocks, cached, schemes) } From fbbfe1ca82602ee396503b2d86189290ec9e97e8 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:54:16 +0100 Subject: [PATCH 16/19] feat: use ledger cost model for storage migration Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/src/host_api/ledger_9.rs | 7 +++-- ledger/src/host_api/migration_8_to_9.rs | 35 ++++++++++++++++++------- pallets/midnight/src/migrations/v2.rs | 21 ++++++++++----- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/ledger/src/host_api/ledger_9.rs b/ledger/src/host_api/ledger_9.rs index ba064a147..669c675ef 100644 --- a/ledger/src/host_api/ledger_9.rs +++ b/ledger/src/host_api/ledger_9.rs @@ -505,11 +505,14 @@ pub trait Ledger9Bridge { /// /// Called by `pallet_midnight`'s v8->v9 storage migration during the runtime /// upgrade that crosses into ledger-9. `state_key` is the pallet's `StateKey` - /// (a v8 arena root); the returned bytes are the new v9 arena root to store back. + /// (a v8 arena root); returns the new v9 arena root to store back, together + /// with the synthetic cost (picoseconds) the translation consumed against + /// the ledger's cost model, for the pallet to charge as this migration's + /// weight. fn migrate_state_v8_to_v9( &mut self, state_key: PassFatPointerAndRead<&[u8]>, - ) -> AllocateAndReturnByCodec, LedgerApiError>> { + ) -> AllocateAndReturnByCodec, u64), LedgerApiError>> { // Ensure the ledger arena is initialized before translating. The migration // runs in the Executive migrations tuple, before pallet_midnight's // on_initialize/on_runtime_upgrade have (re)initialized storage this block. diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs index 8ceaead7a..a878ad6d9 100644 --- a/ledger/src/host_api/migration_8_to_9.rs +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -52,16 +52,26 @@ const LOG_TARGET: &str = "midnight::ledger::migration_8_to_9"; /// Picoseconds granted to each `TypedTranslationState::run` step. The migration /// is single-block (it must complete within one host call), so we loop over -/// `run` with a generous per-step budget until the translation reports a result. -/// One second of picoseconds per step comfortably drains dev/undeployed-sized -/// state in a couple of iterations; the loop cap is a runaway backstop only. -const RUN_BUDGET_PS: u64 = 1_000_000_000_000; +/// `run` with a per-step budget until the translation reports a result. +/// +/// This budget also doubles as the quantization granularity of the synthetic +/// cost reported back to the pallet (see `consumed_cost_ps` below): `run` +/// doesn't return unused budget, so every completed step is charged in full. +/// 10ms keeps that over-approximation small relative to real translation +/// costs while still comfortably draining dev/undeployed-sized state in a +/// handful of iterations; the loop cap is a runaway backstop only. +const RUN_BUDGET_PS: u64 = 10_000_000_000; const MAX_STEPS: usize = 100_000; /// Translate the ledger state referenced by a v8 arena root (`state_key_v8`, -/// the pallet's `StateKey` bytes) into a v9 ledger state, persist it, and return -/// the new v9 arena root to store back into `StateKey`. -pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, LedgerApiError> { +/// the pallet's `StateKey` bytes) into a v9 ledger state, persist it, and +/// return the new v9 arena root (to store back into `StateKey`) together with +/// the synthetic cost, in picoseconds, the translation consumed against the +/// ledger's deterministic cost model — for the pallet to charge as this +/// migration's weight. +pub fn migrate_state_v8_to_v9( + state_key_v8: &[u8], +) -> Result<(Vec, u64), LedgerApiError> { let t_total = std::time::Instant::now(); // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. @@ -108,9 +118,16 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led break result; } }; + // Every completed `run` call before the last one must have exhausted its + // full `RUN_BUDGET_PS` — otherwise `tl.result()` would already have been + // `Some` and the loop would have stopped there. Only the final call may + // have used less. `steps * RUN_BUDGET_PS` is therefore a deterministic + // upper bound on the synthetic cost actually consumed, accurate to one + // `RUN_BUDGET_PS` quantum. + let consumed_cost_ps = steps as u64 * RUN_BUDGET_PS; log::info!( target: LOG_TARGET, - "v8->v9 ledger state translation complete in {steps} step(s), {:?}", + "v8->v9 ledger state translation complete in {steps} step(s), {:?}, {consumed_cost_ps}ps synthetic cost", t_translate.elapsed() ); @@ -135,7 +152,7 @@ pub fn migrate_state_v8_to_v9(state_key_v8: &[u8]) -> Result, Led })?; log::debug!(target: LOG_TARGET, "[perf] migrate_state_v8_to_v9 took {:?}", t_total.elapsed()); - Ok(bytes) + Ok((bytes, consumed_cost_ps)) } #[cfg(test)] diff --git a/pallets/midnight/src/migrations/v2.rs b/pallets/midnight/src/migrations/v2.rs index e04a79c86..72f4c2b56 100644 --- a/pallets/midnight/src/migrations/v2.rs +++ b/pallets/midnight/src/migrations/v2.rs @@ -51,21 +51,30 @@ impl UncheckedOnRuntimeUpgrade for InnerMigrateV1ToV2 { let state_key = StateKey::::get(); // The host function reads the v8 arena root, translates the referenced - // `LedgerState` to v9, re-persists it, and returns the new v9 root. + // `LedgerState` to v9, re-persists it, and returns the new v9 root + // together with the synthetic cost (picoseconds) it charged the + // translation against the ledger's deterministic cost model. // Failure here is unrecoverable: the chain would be left pointing at a // v8 state a ledger-9 runtime cannot read, so we abort the upgrade. - let new_state_key = LedgerApi::migrate_state_v8_to_v9(&state_key) + let (new_state_key, consumed_cost_ps) = LedgerApi::migrate_state_v8_to_v9(&state_key) .expect("FATAL: ledger v8->v9 state migration failed"); StateKey::::put(new_state_key); log::info!( target: "midnight::migration", - "ledger v8->v9 state migration complete; StateKey re-pointed to v9 root" + "ledger v8->v9 state migration complete; StateKey re-pointed to v9 root ({consumed_cost_ps}ps synthetic cost)" ); - // The heavy translation runs natively in the host call; on-chain this is - // a single `StateKey` read + write. - T::DbWeight::get().reads_writes(1, 1) + // The translation runs natively in the host call, so the pallet-side + // read/write above is negligible next to it; report the ledger's own + // synthetic cost as `ref_time` instead. This is the same 1:1 mapping + // `pallet_midnight::get_tx_weight` uses for ordinary transactions + // (ledger picoseconds -> `Weight` ref_time), and that cost model + // already prices the arena reads/writes the translation performs, so + // no separate `DbWeight` charge is added on top. `proof_size` is 0: + // this chain doesn't build a PoV. Capped at the block's max weight, + // since a pathological state could in principle exceed it. + Weight::from_parts(consumed_cost_ps, 0).min(T::BlockWeights::get().max_block) } #[cfg(feature = "try-runtime")] From 7c16094dd2b2f72528be136a6351317fab8e9613 Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:06:09 +0100 Subject: [PATCH 17/19] chore(local-environment): npm audit fix Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- local-environment/package-lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/local-environment/package-lock.json b/local-environment/package-lock.json index ce6bae903..9a0bead50 100644 --- a/local-environment/package-lock.json +++ b/local-environment/package-lock.json @@ -1552,15 +1552,15 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/call-bind-apply-helpers": { From 2027d5ec85ec620d5b099be20127358f7c55535b Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:18:01 +0100 Subject: [PATCH 18/19] fix: no-op ledger v8->v9 migration when state is already ledger-9 The 2.0.0 runtime runs ledger-9 but shipped pallet-midnight at storage version 1 (it had no v1->v2 migration), so a network upgrading 2.0.0 -> this runtime still fires VersionedMigration<1,2> over an already-v9 StateKey. Feeding that v9 root to the v8 decode path fails on the tag mismatch and the pallet's expect() panics, bricking the upgrade block. Guard migrate_state_v8_to_v9 so it no-ops when the StateKey already references a ledger-9 state: detect it by the arena root's serialized tag (distinct between ledger-state[v13] and [v18]) and return the key unchanged with zero synthetic cost, leaving only the storage-version bump. Assisted-by: Claude:claude-fable-5 Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- ledger/src/host_api/migration_8_to_9.rs | 25 +++++++++++++++++++++++++ pallets/midnight/src/migrations/v2.rs | 24 +++++++++++++++++------- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/ledger/src/host_api/migration_8_to_9.rs b/ledger/src/host_api/migration_8_to_9.rs index a878ad6d9..6fe05c873 100644 --- a/ledger/src/host_api/migration_8_to_9.rs +++ b/ledger/src/host_api/migration_8_to_9.rs @@ -69,11 +69,36 @@ const MAX_STEPS: usize = 100_000; /// the synthetic cost, in picoseconds, the translation consumed against the /// ledger's deterministic cost model — for the pallet to charge as this /// migration's weight. +/// +/// If `state_key_v8` already references a ledger-9 state this is a no-op: it +/// returns the key unchanged with zero cost (see the idempotency guard below). pub fn migrate_state_v8_to_v9( state_key_v8: &[u8], ) -> Result<(Vec, u64), LedgerApiError> { let t_total = std::time::Instant::now(); + // 0. Idempotency guard: no-op if the state is already ledger-9. + // + // The pallet gates this behind `VersionedMigration<1, 2>`, but the on-chain + // pallet-midnight storage version is not a faithful proxy for the ledger + // version. The 2.0.0 runtime (spec 2_000_000) already runs ledger-9 yet + // shipped pallet-midnight at storage version 1 (it had no v1->v2 migration), + // so a network upgrading 2.0.0 -> this runtime still triggers this migration + // even though its `StateKey` already points at a v9 arena root. Feeding that + // v9 root to the v8 decode below would fail on the tag mismatch and abort the + // upgrade (the pallet `expect`s success), bricking the chain. Detect it by + // the root's serialized tag — `TypedArenaKey`'s tag embeds the `LedgerState` + // version (`storage-key(midnight:ledger-state[vN]:...)`), so a successful + // tagged decode as a v9 key is a reliable, arena-free discriminator — and + // return the key unchanged with zero synthetic cost. + if tagged_deserialize::, D::Hasher>>(&mut &state_key_v8[..]).is_ok() { + log::info!( + target: LOG_TARGET, + "StateKey already references a ledger-9 state; skipping v8->v9 translation (no-op)" + ); + return Ok((state_key_v8.to_vec(), 0)); + } + // 1. Decode the v8 arena root and load the v8 ledger wrapper from the arena. let t_load = std::time::Instant::now(); let key8: TypedArenaKey, D::Hasher> = tagged_deserialize(&mut &state_key_v8[..]) diff --git a/pallets/midnight/src/migrations/v2.rs b/pallets/midnight/src/migrations/v2.rs index 72f4c2b56..e9154eca7 100644 --- a/pallets/midnight/src/migrations/v2.rs +++ b/pallets/midnight/src/migrations/v2.rs @@ -26,10 +26,17 @@ //! It is a single-block migration: the host call translates the whole state in //! one shot (a few milliseconds for dev/undeployed-sized state). It is wired //! into [`crate::Migrations`](../../../runtime/src/lib.rs) via -//! [`VersionedMigration`], so it runs exactly once, when a ledger-8 runtime -//! (pallet-midnight storage version 1) upgrades to this ledger-9 runtime -//! (storage version 2). A chain whose genesis is already ledger-9 starts at -//! storage version 2 and never runs it. +//! [`VersionedMigration`], so it runs at most once, when a runtime at +//! pallet-midnight storage version 1 upgrades to this runtime (storage version +//! 2). A chain whose genesis is already ledger-9 starts at storage version 2 +//! and never runs it. +//! +//! Storage version 1 does not, however, imply the *ledger* state is still v8: +//! the 2.0.0 runtime already ran ledger-9 yet shipped pallet-midnight at storage +//! version 1 (it had no v1->v2 migration), so a network upgrading 2.0.0 -> this +//! runtime triggers this migration over an already-v9 state. The host function +//! detects that case and no-ops (returns the `StateKey` unchanged), so the only +//! effect on that path is the storage-version bump to 2. #[cfg(feature = "try-runtime")] extern crate alloc; @@ -53,9 +60,12 @@ impl UncheckedOnRuntimeUpgrade for InnerMigrateV1ToV2 { // The host function reads the v8 arena root, translates the referenced // `LedgerState` to v9, re-persists it, and returns the new v9 root // together with the synthetic cost (picoseconds) it charged the - // translation against the ledger's deterministic cost model. - // Failure here is unrecoverable: the chain would be left pointing at a - // v8 state a ledger-9 runtime cannot read, so we abort the upgrade. + // translation against the ledger's deterministic cost model. If the + // state is already v9 (e.g. a 2.0.0 -> this-runtime upgrade), it no-ops + // and returns the `state_key` unchanged with zero cost. + // A genuine failure here is unrecoverable: the chain would be left + // pointing at a v8 state a ledger-9 runtime cannot read, so we abort the + // upgrade. let (new_state_key, consumed_cost_ps) = LedgerApi::migrate_state_v8_to_v9(&state_key) .expect("FATAL: ledger v8->v9 state migration failed"); From 1b82adb405becffd2e5cdfed090c8eb545a54c3a Mon Sep 17 00:00:00 2001 From: Oscar Bailey <79094698+ozgb@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:42:35 +0100 Subject: [PATCH 19/19] docs: update change file Signed-off-by: Oscar Bailey <79094698+ozgb@users.noreply.github.com> --- changes/runtime/changed/ledger-8-to-9-hardfork-migration.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md index 6bb3b15ab..e2c92f72c 100644 --- a/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md +++ b/changes/runtime/changed/ledger-8-to-9-hardfork-migration.md @@ -9,7 +9,8 @@ the on-chain `LedgerState` from v13 to v18. It's wired in as `pallet_midnight::migrations::v2::MigrateV1ToV2`, a `VersionedMigration<1,2,..>` that fires once when a ledger-8 chain (pallet-midnight storage version 1) upgrades to this runtime (storage version 2); a fresh ledger-9 genesis starts -at version 2 and skips it. +at version 2 and skips it. The migration's weight is derived from the ledger +cost model rather than a hand-tuned estimate. Also includes two fixes needed to support both ledger-8 and ledger-9 chains: version-aware genesis seeding (detected via `serialize::peek_tag` instead of