From b8d122cc910014500aaacf535eeae4aae998260f Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Fri, 17 Jul 2026 10:24:35 -0700 Subject: [PATCH 01/21] Add Aptos DB fork tool --- Cargo.lock | 4 + crates/aptos-genesis/src/builder.rs | 34 +- storage/db-tool/Cargo.toml | 4 + storage/db-tool/src/fork.rs | 520 ++++++++++++++++++ storage/db-tool/src/lib.rs | 4 + .../src/account_config/resources/chain_id.rs | 10 +- types/src/on_chain_config/mod.rs | 8 +- 7 files changed, 579 insertions(+), 5 deletions(-) create mode 100644 storage/db-tool/src/fork.rs diff --git a/Cargo.lock b/Cargo.lock index 707cad4fb01..c10b7e107ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1195,12 +1195,14 @@ dependencies = [ "aptos-backup-cli", "aptos-backup-service", "aptos-block-executor", + "aptos-cached-packages", "aptos-config", "aptos-db", "aptos-db-indexer", "aptos-executor", "aptos-executor-test-helpers", "aptos-executor-types", + "aptos-genesis", "aptos-indexer-grpc-table-info", "aptos-logger", "aptos-storage-interface", @@ -1210,6 +1212,8 @@ dependencies = [ "bcs 0.1.4", "clap 4.5.21", "itertools 0.13.0", + "move-core-types", + "rand 0.7.3", "rayon", "serde_json", "serial_test", diff --git a/crates/aptos-genesis/src/builder.rs b/crates/aptos-genesis/src/builder.rs index c3440d5cee2..a5afea864d3 100644 --- a/crates/aptos-genesis/src/builder.rs +++ b/crates/aptos-genesis/src/builder.rs @@ -31,9 +31,11 @@ use aptos_types::{ keyless::Groth16VerificationKey, on_chain_config::{ Features, GasScheduleV2, OnChainConsensusConfig, OnChainExecutionConfig, - OnChainJWKConsensusConfig, OnChainRandomnessConfig, + OnChainJWKConsensusConfig, OnChainRandomnessConfig, ValidatorSet, }, transaction::Transaction, + validator_config::ValidatorConfig, + validator_info::ValidatorInfo, waypoint::Waypoint, }; use aptos_vm_genesis::default_gas_schedule; @@ -155,6 +157,26 @@ impl ValidatorNodeConfig { } } + pub fn validator_info(&self) -> anyhow::Result { + let (_, _, private_identity, _) = self.get_key_objects(None)?; + let validator_configuration: ValidatorConfiguration = self.try_into()?; + let validator: aptos_vm_genesis::Validator = validator_configuration.try_into()?; + Ok(ValidatorInfo::new( + validator.owner_address, + validator.stake_amount, + ValidatorConfig::new( + private_identity.consensus_private_key.public_key(), + validator.network_addresses, + validator.full_node_network_addresses, + self.index as u64, + ), + )) + } + + pub fn validator_config_path(&self) -> PathBuf { + self.dir.join(CONFIG_FILE) + } + fn insert_genesis(&mut self, genesis: &Transaction) { let config = self.config.override_config_mut(); config.execution.genesis = Some(genesis.clone()); @@ -189,6 +211,16 @@ impl ValidatorNodeConfig { } } +pub fn validator_set_from_local_validators( + validators: &[ValidatorNodeConfig], +) -> anyhow::Result { + validators + .iter() + .map(ValidatorNodeConfig::validator_info) + .collect::>>() + .map(ValidatorSet::new) +} + impl TryFrom<&ValidatorNodeConfig> for ValidatorConfiguration { type Error = anyhow::Error; diff --git a/storage/db-tool/Cargo.toml b/storage/db-tool/Cargo.toml index a7ebbe0f6b7..7e5f23672b4 100644 --- a/storage/db-tool/Cargo.toml +++ b/storage/db-tool/Cargo.toml @@ -15,11 +15,13 @@ rust-version = { workspace = true } anyhow = { workspace = true } aptos-backup-cli = { workspace = true } aptos-block-executor = { workspace = true } +aptos-cached-packages = { workspace = true } aptos-config = { workspace = true } aptos-db = { workspace = true, features = ["db-debugger"] } aptos-db-indexer = { workspace = true } aptos-executor = { workspace = true } aptos-executor-types = { workspace = true } +aptos-genesis = { workspace = true } aptos-logger = { workspace = true } aptos-storage-interface = { workspace = true } aptos-temppath = { workspace = true } @@ -28,6 +30,8 @@ aptos-vm = { workspace = true } bcs = { workspace = true } clap = { workspace = true } itertools = { workspace = true } +move-core-types = { workspace = true } +rand = { workspace = true } rayon = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs new file mode 100644 index 00000000000..ea4dc6a5e3f --- /dev/null +++ b/storage/db-tool/src/fork.rs @@ -0,0 +1,520 @@ +// Copyright © Aptos Foundation +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{ensure, format_err, Context, Result}; +use aptos_config::config::{ + InitialSafetyRulesConfig, NodeConfig, RocksdbConfigs, StorageDirPaths, WaypointConfig, + BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, + NO_OP_STORAGE_PRUNER_CONFIG, +}; +use aptos_db::AptosDB; +use aptos_executor::db_bootstrapper::{calculate_genesis, generate_waypoint}; +use aptos_genesis::builder::{validator_set_from_local_validators, Builder, ValidatorNodeConfig}; +use aptos_storage_interface::{ + state_store::state_view::db_state_view::LatestDbStateCheckpointView, DbReaderWriter, +}; +use aptos_types::{ + account_address::AccountAddress, + account_config::{ + new_block_event_key, ChainIdResource, NewBlockEvent, NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG, + }, + chain_id::ChainId, + contract_event::ContractEvent, + on_chain_config::{ConfigurationResource, OnChainConfig, ValidatorSet}, + state_store::state_key::StateKey, + transaction::{ChangeSet, Transaction, WriteSetPayload}, + waypoint::Waypoint, + write_set::{WriteOp, WriteSetMut}, +}; +use aptos_vm::aptos_vm::AptosVMBlockExecutor; +use clap::Parser; +use move_core_types::{language_storage::TypeTag, move_resource::MoveStructType}; +use rand::rngs::OsRng; +use serde_json::json; +use std::{ + fs::{self, File}, + io::Write, + num::NonZeroUsize, + path::{Path, PathBuf}, +}; + +const GENESIS_BLOB: &str = "genesis.blob"; +const MANIFEST: &str = "fork-manifest.json"; +const VALIDATOR_IDENTITY: &str = "validator-identity.yaml"; + +#[derive(Parser)] +#[clap( + name = "aptos-db-fork", + about = "Copy a local Aptos DB and commit a fork reconfiguration with a new chain ID and local validator set." +)] +pub struct Command { + #[clap(long, value_parser)] + source_db_dir: PathBuf, + + #[clap(long, value_parser)] + output_db_dir: PathBuf, + + #[clap(long, value_parser)] + config_dir: Option, + + #[clap(long, value_parser)] + fork_chain_id: u8, + + #[clap(long, default_value_t = 1)] + validators: usize, + + #[clap(long)] + enable_storage_sharding: bool, +} + +impl Command { + pub fn run(self) -> Result<()> { + let validators = NonZeroUsize::new(self.validators) + .ok_or_else(|| format_err!("--validators must be greater than 0"))?; + let fork_chain_id = validate_requested_chain_id(self.fork_chain_id)?; + validate_output_paths(&self.source_db_dir, &self.output_db_dir)?; + let config_dir = self + .config_dir + .clone() + .unwrap_or_else(|| default_config_dir(&self.output_db_dir)); + validate_config_path(&self.source_db_dir, &self.output_db_dir, &config_dir)?; + + let source_info = inspect_db(&self.source_db_dir, self.enable_storage_sharding) + .with_context(|| format!("failed to inspect source DB {:?}", self.source_db_dir))?; + ensure!( + fork_chain_id != source_info.chain_id, + "fork chain ID must differ from source chain ID {}", + source_info.chain_id.id(), + ); + + fs::create_dir_all(&self.output_db_dir) + .with_context(|| format!("failed to create {:?}", self.output_db_dir))?; + AptosDB::create_checkpoint( + &self.source_db_dir, + &self.output_db_dir, + self.enable_storage_sharding, + ) + .with_context(|| { + format!( + "failed to create checkpoint from {:?} to {:?}", + self.source_db_dir, self.output_db_dir + ) + })?; + + fs::create_dir_all(&config_dir) + .with_context(|| format!("failed to create config dir {:?}", config_dir))?; + let (_root_key, _generated_genesis, _generated_waypoint, validators) = Builder::new( + &config_dir, + aptos_cached_packages::head_release_bundle().clone(), + )? + .with_num_validators(validators) + .build(OsRng)?; + let validator_set = validator_set_from_local_validators(&validators)?; + let generated_validator_addresses = validator_set.active_validators(); + + let (fork_txn, waypoint) = commit_fork_transaction( + &self.output_db_dir, + self.enable_storage_sharding, + fork_chain_id, + &validator_set, + )?; + + let output_info = inspect_db(&self.output_db_dir, self.enable_storage_sharding) + .with_context(|| format!("failed to inspect output DB {:?}", self.output_db_dir))?; + ensure!( + output_info.chain_id == fork_chain_id, + "output chain ID readback mismatch: expected {}, got {}", + fork_chain_id.id(), + output_info.chain_id.id(), + ); + ensure!( + output_info.validator_set == validator_set, + "output ValidatorSet readback mismatch", + ); + ensure!( + output_info.waypoint == waypoint, + "output waypoint readback mismatch: manifest {}, DB {}", + waypoint, + output_info.waypoint, + ); + ensure!( + output_info.validator_set != source_info.validator_set, + "fork did not replace ValidatorSet", + ); + + let node_config_paths = write_fork_configs( + &validators, + &self.output_db_dir, + &fork_txn, + waypoint, + fork_chain_id, + )?; + let manifest_path = write_manifest( + &config_dir, + &self.source_db_dir, + &self.output_db_dir, + &node_config_paths, + &source_info, + &output_info, + fork_chain_id, + &generated_validator_addresses, + )?; + + println!("Fork DB written to: {}", self.output_db_dir.display()); + println!("Fork waypoint: {}", waypoint); + println!("Fork manifest: {}", manifest_path.display()); + Ok(()) + } +} + +#[derive(Clone)] +struct DbInfo { + version: u64, + chain_id: ChainId, + validator_set: ValidatorSet, + waypoint: Waypoint, + accumulator_root: String, + state_root: String, + epoch: u64, + next_epoch: Option, +} + +fn validate_requested_chain_id(chain_id: u8) -> Result { + ensure!( + chain_id != 0 && chain_id != 1, + "fork chain ID must not be 0 or mainnet (1)", + ); + Ok(ChainId::new(chain_id)) +} + +fn validate_output_paths(source_db_dir: &Path, output_db_dir: &Path) -> Result<()> { + ensure!(source_db_dir.exists(), "source DB dir does not exist"); + ensure!( + !output_db_dir.exists(), + "output DB dir already exists; refusing to mutate an existing path", + ); + let source = source_db_dir.canonicalize()?; + let output_parent = output_db_dir + .parent() + .unwrap_or_else(|| Path::new(".")) + .canonicalize()?; + let output_name = output_db_dir + .file_name() + .ok_or_else(|| format_err!("output DB dir must name a child path"))?; + let output = output_parent.join(output_name); + ensure!(source != output, "source and output DB paths must differ"); + ensure!( + !output.starts_with(&source) && !source.starts_with(&output), + "source and output DB paths must not be nested", + ); + Ok(()) +} + +fn validate_config_path( + source_db_dir: &Path, + output_db_dir: &Path, + config_dir: &Path, +) -> Result<()> { + if config_dir.exists() { + ensure!( + config_dir.read_dir()?.next().is_none(), + "config dir already exists and is not empty", + ); + } + let source = source_db_dir.canonicalize()?; + let output = output_db_dir + .parent() + .unwrap_or_else(|| Path::new(".")) + .canonicalize()? + .join( + output_db_dir + .file_name() + .ok_or_else(|| format_err!("output DB dir must name a child path"))?, + ); + let config_parent = config_dir + .parent() + .unwrap_or_else(|| Path::new(".")) + .canonicalize()?; + let config_name = config_dir + .file_name() + .ok_or_else(|| format_err!("config dir must name a child path"))?; + let config = config_parent.join(config_name); + ensure!( + !config.starts_with(&source), + "config dir must not be nested under source DB dir", + ); + ensure!( + !config.starts_with(&output), + "config dir must not be nested under output DB dir", + ); + Ok(()) +} + +fn default_config_dir(output_db_dir: &Path) -> PathBuf { + let name = output_db_dir + .file_name() + .map(|name| format!("{}-configs", name.to_string_lossy())) + .unwrap_or_else(|| "fork-configs".to_string()); + output_db_dir + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(name) +} + +fn inspect_db(db_dir: &Path, sharding: bool) -> Result { + let db = open_db(db_dir, true, sharding)?; + let view = db.reader.latest_state_checkpoint_view()?; + let chain_id = ChainIdResource::fetch_config(&view) + .ok_or_else(|| format_err!("ChainIdResource missing"))? + .chain_id(); + let validator_set = + ValidatorSet::fetch_config(&view).ok_or_else(|| format_err!("ValidatorSet missing"))?; + let ledger_info = db.reader.get_latest_ledger_info()?; + let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; + let waypoint = Waypoint::new_epoch_boundary(ledger_info.ledger_info())?; + Ok(DbInfo { + version: ledger_info.ledger_info().version(), + chain_id, + validator_set, + waypoint, + accumulator_root: ledger_info + .ledger_info() + .commit_info() + .executed_state_id() + .to_string(), + state_root: ledger_summary.state_summary.root_hash().to_string(), + epoch: ledger_info.ledger_info().epoch(), + next_epoch: ledger_info + .ledger_info() + .next_epoch_state() + .map(|state| state.epoch), + }) +} + +fn commit_fork_transaction( + output_db_dir: &Path, + sharding: bool, + fork_chain_id: ChainId, + validator_set: &ValidatorSet, +) -> Result<(Transaction, Waypoint)> { + let db = open_db(output_db_dir, false, sharding)?; + let view = db.reader.latest_state_checkpoint_view()?; + let configuration = ConfigurationResource::fetch_config(&view) + .ok_or_else(|| format_err!("ConfigurationResource missing"))?; + let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; + let fork_txn = Transaction::GenesisTransaction(WriteSetPayload::Direct(ChangeSet::new( + WriteSetMut::new(vec![ + ( + StateKey::on_chain_config::()?, + WriteOp::legacy_modification( + bcs::to_bytes(&ChainIdResource::new(fork_chain_id))?.into(), + ), + ), + ( + StateKey::on_chain_config::()?, + WriteOp::legacy_modification(bcs::to_bytes(validator_set)?.into()), + ), + ( + StateKey::on_chain_config::()?, + WriteOp::legacy_modification( + bcs::to_bytes(&configuration.bump_epoch_for_reconfiguration())?.into(), + ), + ), + ]) + .freeze()?, + vec![ + ContractEvent::new_v2(NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG.clone(), vec![])?, + ContractEvent::new_v1( + new_block_event_key(), + 0, + TypeTag::Struct(Box::new(NewBlockEvent::struct_tag())), + vec![], + )?, + ], + ))); + let waypoint = generate_waypoint::(&db, &fork_txn)?; + let committer = calculate_genesis::(&db, ledger_summary, &fork_txn)?; + ensure!( + waypoint == committer.waypoint(), + "fork waypoint changed between calculation passes", + ); + committer.commit()?; + Ok((fork_txn, waypoint)) +} + +fn open_db(db_dir: &Path, readonly: bool, sharding: bool) -> Result { + let rocksdb_configs = RocksdbConfigs { + enable_storage_sharding: sharding, + ..Default::default() + }; + let db = AptosDB::open( + StorageDirPaths::from_path(db_dir), + readonly, + NO_OP_STORAGE_PRUNER_CONFIG, + rocksdb_configs, + false, + BUFFERED_STATE_TARGET_ITEMS, + DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, + None, + )?; + Ok(DbReaderWriter::new(db)) +} + +fn write_fork_configs( + validators: &[ValidatorNodeConfig], + output_db_dir: &Path, + fork_txn: &Transaction, + waypoint: Waypoint, + fork_chain_id: ChainId, +) -> Result> { + validators + .iter() + .map(|validator| { + let config_path = validator.validator_config_path(); + let mut config = NodeConfig::load_from_path(&config_path)?; + config.storage.dir = output_db_dir.to_path_buf(); + config.base.waypoint = WaypointConfig::FromConfig(waypoint); + config.execution.genesis = Some(fork_txn.clone()); + config.execution.genesis_waypoint = Some(WaypointConfig::FromConfig(waypoint)); + config.execution.genesis_file_location = validator.dir.join(GENESIS_BLOB); + config.consensus.safety_rules.initial_safety_rules_config = + InitialSafetyRulesConfig::from_file( + validator.dir.join(VALIDATOR_IDENTITY), + vec![], + WaypointConfig::FromConfig(waypoint), + ); + File::create(&config.execution.genesis_file_location)? + .write_all(&bcs::to_bytes(fork_txn)?)?; + config.save_to_path(&config_path)?; + fs::write( + validator.dir.join("chain_id"), + fork_chain_id.id().to_string(), + )?; + Ok(config_path) + }) + .collect() +} + +fn write_manifest( + config_dir: &Path, + source_db_dir: &Path, + output_db_dir: &Path, + node_config_paths: &[PathBuf], + source_info: &DbInfo, + output_info: &DbInfo, + fork_chain_id: ChainId, + validator_addresses: &[AccountAddress], +) -> Result { + let manifest = json!({ + "source_db_dir": source_db_dir, + "output_db_dir": output_db_dir, + "source_ledger": ledger_json(source_info), + "output_ledger": ledger_json(output_info), + "chain_id_change": { + "source": source_info.chain_id.id(), + "fork": fork_chain_id.id(), + }, + "validator_replacement": { + "source_validator_count": source_info.validator_set.num_validators(), + "fork_validator_count": output_info.validator_set.num_validators(), + "source_validator_addresses": source_info.validator_set.active_validators(), + "fork_validator_addresses": validator_addresses, + }, + "generated_config_paths": node_config_paths, + "waypoint": output_info.waypoint.to_string(), + }); + let manifest_path = config_dir.join(MANIFEST); + fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; + Ok(manifest_path) +} + +fn ledger_json(info: &DbInfo) -> serde_json::Value { + json!({ + "version": info.version, + "epoch": info.epoch, + "next_epoch": info.next_epoch, + "waypoint": info.waypoint.to_string(), + "accumulator_root": info.accumulator_root, + "state_root": info.state_root, + "chain_id": info.chain_id.id(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use aptos_executor_test_helpers::bootstrap_genesis; + use aptos_temppath::TempPath; + + #[test] + fn rejects_reserved_chain_ids() { + assert!(validate_requested_chain_id(0).is_err()); + assert!(validate_requested_chain_id(1).is_err()); + assert_eq!(validate_requested_chain_id(42).unwrap().id(), 42); + } + + #[test] + fn rejects_aliasing_and_nested_output_paths() { + let root = TempPath::new(); + root.create_as_dir().unwrap(); + let source = root.path().join("source"); + fs::create_dir_all(&source).unwrap(); + assert!(validate_output_paths(&source, &source).is_err()); + assert!(validate_output_paths(&source, &source.join("child")).is_err()); + assert!(validate_output_paths(&source, &root.path().join("output")).is_ok()); + } + + #[test] + fn fork_commits_chain_id_validator_set_and_waypoint_to_output_db() { + let source_db_dir = TempPath::new(); + source_db_dir.create_as_dir().unwrap(); + let source_config_dir = TempPath::new(); + source_config_dir.create_as_dir().unwrap(); + let (_root_key, genesis, _waypoint, _validators) = Builder::new( + source_config_dir.path(), + aptos_cached_packages::head_release_bundle().clone(), + ) + .unwrap() + .with_num_validators(NonZeroUsize::new(1).unwrap()) + .build(OsRng) + .unwrap(); + + { + let source_db = DbReaderWriter::new(AptosDB::new_for_test(source_db_dir.path())); + bootstrap_genesis::(&source_db, &genesis).unwrap(); + } + + let source_info_before = inspect_db(source_db_dir.path(), false).unwrap(); + let output_root = TempPath::new(); + output_root.create_as_dir().unwrap(); + let output_db_dir = output_root.path().join("fork-db"); + let config_dir = output_root.path().join("fork-configs"); + let fork_chain_id = ChainId::new(42); + + Command { + source_db_dir: source_db_dir.path().to_path_buf(), + output_db_dir: output_db_dir.clone(), + config_dir: Some(config_dir.clone()), + fork_chain_id: fork_chain_id.id(), + validators: 1, + enable_storage_sharding: false, + } + .run() + .unwrap(); + + let source_info_after = inspect_db(source_db_dir.path(), false).unwrap(); + let output_info = inspect_db(&output_db_dir, false).unwrap(); + assert_eq!(source_info_after.chain_id, source_info_before.chain_id); + assert_eq!( + source_info_after.validator_set, + source_info_before.validator_set + ); + assert_eq!(output_info.chain_id, fork_chain_id); + assert_ne!(output_info.chain_id, source_info_before.chain_id); + assert_ne!(output_info.validator_set, source_info_before.validator_set); + assert_eq!(output_info.validator_set.num_validators(), 1); + assert_eq!(output_info.waypoint.version(), source_info_before.version + 1); + assert!(output_info.next_epoch.is_some()); + assert!(config_dir.join(MANIFEST).exists()); + } +} diff --git a/storage/db-tool/src/lib.rs b/storage/db-tool/src/lib.rs index b777fa1d0d8..5e22b759c2e 100644 --- a/storage/db-tool/src/lib.rs +++ b/storage/db-tool/src/lib.rs @@ -6,6 +6,7 @@ extern crate core; mod backup; mod backup_maintenance; mod bootstrap; +mod fork; mod gen_replay_verify_jobs; mod replay_on_archive; mod replay_verify; @@ -30,6 +31,8 @@ pub enum DBTool { Bootstrap(bootstrap::Command), + Fork(fork::Command), + #[clap(subcommand)] Debug(db_debugger::Cmd), @@ -49,6 +52,7 @@ impl DBTool { DBTool::Backup(cmd) => cmd.run().await, DBTool::BackupMaintenance(cmd) => cmd.run().await, DBTool::Bootstrap(cmd) => cmd.run(), + DBTool::Fork(cmd) => cmd.run(), DBTool::Debug(cmd) => Ok(cmd.run()?), DBTool::ReplayVerify(cmd) => { let ret = cmd.run().await; diff --git a/types/src/account_config/resources/chain_id.rs b/types/src/account_config/resources/chain_id.rs index 37b9f26b19e..67e030d753d 100644 --- a/types/src/account_config/resources/chain_id.rs +++ b/types/src/account_config/resources/chain_id.rs @@ -8,14 +8,20 @@ use move_core_types::{ identifier::IdentStr, move_resource::{MoveResource, MoveStructType}, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(Deserialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct ChainIdResource { chain_id: u8, } impl ChainIdResource { + pub fn new(chain_id: ChainId) -> Self { + Self { + chain_id: chain_id.id(), + } + } + pub fn chain_id(&self) -> ChainId { ChainId::new(self.chain_id) } diff --git a/types/src/on_chain_config/mod.rs b/types/src/on_chain_config/mod.rs index a69a81eb220..3ed248166d8 100644 --- a/types/src/on_chain_config/mod.rs +++ b/types/src/on_chain_config/mod.rs @@ -257,8 +257,7 @@ impl ConfigurationResource { &self.events } - #[cfg(feature = "fuzzing")] - pub fn bump_epoch_for_test(&self) -> Self { + pub fn bump_epoch_for_reconfiguration(&self) -> Self { let epoch = self.epoch + 1; let last_reconfiguration_time = self.last_reconfiguration_time + 1; let mut events = self.events.clone(); @@ -271,6 +270,11 @@ impl ConfigurationResource { } } + #[cfg(feature = "fuzzing")] + pub fn bump_epoch_for_test(&self) -> Self { + self.bump_epoch_for_reconfiguration() + } + #[cfg(feature = "fuzzing")] pub fn set_last_reconfiguration_time_for_test(&mut self, last_reconfiguration_time: u64) { self.last_reconfiguration_time = last_reconfiguration_time; From 4db76913cd71c3dfc59d4c9f3ebd8f3f485198c4 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Fri, 17 Jul 2026 10:38:05 -0700 Subject: [PATCH 02/21] Fix fork tool source and event guards --- Cargo.lock | 1 + storage/db-tool/Cargo.toml | 1 + storage/db-tool/src/fork.rs | 198 ++++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c10b7e107ee..0a7d569aabb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,6 +1197,7 @@ dependencies = [ "aptos-block-executor", "aptos-cached-packages", "aptos-config", + "aptos-crypto", "aptos-db", "aptos-db-indexer", "aptos-executor", diff --git a/storage/db-tool/Cargo.toml b/storage/db-tool/Cargo.toml index 7e5f23672b4..80f361cdd19 100644 --- a/storage/db-tool/Cargo.toml +++ b/storage/db-tool/Cargo.toml @@ -39,6 +39,7 @@ tokio = { workspace = true } [dev-dependencies] aptos-backup-cli = { workspace = true, features = ["testing"] } aptos-backup-service = { workspace = true } +aptos-crypto = { workspace = true } aptos-executor-test-helpers = { workspace = true } aptos-indexer-grpc-table-info = { workspace = true } serial_test = { workspace = true } diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index ea4dc6a5e3f..612585ba5f2 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -16,12 +16,13 @@ use aptos_storage_interface::{ use aptos_types::{ account_address::AccountAddress, account_config::{ - new_block_event_key, ChainIdResource, NewBlockEvent, NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG, + new_block_event_key, BlockResource, ChainIdResource, NewBlockEvent, CORE_CODE_ADDRESS, + NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG, }, chain_id::ChainId, contract_event::ContractEvent, on_chain_config::{ConfigurationResource, OnChainConfig, ValidatorSet}, - state_store::state_key::StateKey, + state_store::{state_key::StateKey, TStateView}, transaction::{ChangeSet, Transaction, WriteSetPayload}, waypoint::Waypoint, write_set::{WriteOp, WriteSetMut}, @@ -71,6 +72,10 @@ impl Command { pub fn run(self) -> Result<()> { let validators = NonZeroUsize::new(self.validators) .ok_or_else(|| format_err!("--validators must be greater than 0"))?; + ensure!( + validators.get() == 1, + "--validators > 1 is not supported until per-validator DB output is implemented", + ); let fork_chain_id = validate_requested_chain_id(self.fork_chain_id)?; validate_output_paths(&self.source_db_dir, &self.output_db_dir)?; let config_dir = self @@ -79,7 +84,7 @@ impl Command { .unwrap_or_else(|| default_config_dir(&self.output_db_dir)); validate_config_path(&self.source_db_dir, &self.output_db_dir, &config_dir)?; - let source_info = inspect_db(&self.source_db_dir, self.enable_storage_sharding) + let source_info = inspect_db(&self.source_db_dir, self.enable_storage_sharding, false) .with_context(|| format!("failed to inspect source DB {:?}", self.source_db_dir))?; ensure!( fork_chain_id != source_info.chain_id, @@ -119,7 +124,7 @@ impl Command { &validator_set, )?; - let output_info = inspect_db(&self.output_db_dir, self.enable_storage_sharding) + let output_info = inspect_db(&self.output_db_dir, self.enable_storage_sharding, true) .with_context(|| format!("failed to inspect output DB {:?}", self.output_db_dir))?; ensure!( output_info.chain_id == fork_chain_id, @@ -132,10 +137,13 @@ impl Command { "output ValidatorSet readback mismatch", ); ensure!( - output_info.waypoint == waypoint, + output_info.waypoint == Some(waypoint), "output waypoint readback mismatch: manifest {}, DB {}", waypoint, - output_info.waypoint, + output_info + .waypoint + .map(|waypoint| waypoint.to_string()) + .unwrap_or_else(|| "".to_string()), ); ensure!( output_info.validator_set != source_info.validator_set, @@ -172,11 +180,12 @@ struct DbInfo { version: u64, chain_id: ChainId, validator_set: ValidatorSet, - waypoint: Waypoint, + waypoint: Option, accumulator_root: String, state_root: String, epoch: u64, next_epoch: Option, + new_block_event_count: u64, } fn validate_requested_chain_id(chain_id: u8) -> Result { @@ -261,7 +270,7 @@ fn default_config_dir(output_db_dir: &Path) -> PathBuf { .join(name) } -fn inspect_db(db_dir: &Path, sharding: bool) -> Result { +fn inspect_db(db_dir: &Path, sharding: bool, require_epoch_waypoint: bool) -> Result { let db = open_db(db_dir, true, sharding)?; let view = db.reader.latest_state_checkpoint_view()?; let chain_id = ChainIdResource::fetch_config(&view) @@ -269,9 +278,21 @@ fn inspect_db(db_dir: &Path, sharding: bool) -> Result { .chain_id(); let validator_set = ValidatorSet::fetch_config(&view).ok_or_else(|| format_err!("ValidatorSet missing"))?; + let block_resource = view + .get_state_value_bytes(&StateKey::resource_typed::(&CORE_CODE_ADDRESS)?)? + .ok_or_else(|| format_err!("BlockResource missing"))?; + let block_resource = bcs::from_bytes::(&block_resource)?; let ledger_info = db.reader.get_latest_ledger_info()?; let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; - let waypoint = Waypoint::new_epoch_boundary(ledger_info.ledger_info())?; + let waypoint = if ledger_info.ledger_info().ends_epoch() { + Some(Waypoint::new_epoch_boundary(ledger_info.ledger_info())?) + } else { + ensure!( + !require_epoch_waypoint, + "latest ledger info is not an epoch boundary", + ); + None + }; Ok(DbInfo { version: ledger_info.ledger_info().version(), chain_id, @@ -288,6 +309,7 @@ fn inspect_db(db_dir: &Path, sharding: bool) -> Result { .ledger_info() .next_epoch_state() .map(|state| state.epoch), + new_block_event_count: block_resource.new_block_events().count(), }) } @@ -301,6 +323,21 @@ fn commit_fork_transaction( let view = db.reader.latest_state_checkpoint_view()?; let configuration = ConfigurationResource::fetch_config(&view) .ok_or_else(|| format_err!("ConfigurationResource missing"))?; + let block_resource = view + .get_state_value_bytes(&StateKey::resource_typed::(&CORE_CODE_ADDRESS)?)? + .ok_or_else(|| format_err!("BlockResource missing"))?; + let block_resource = bcs::from_bytes::(&block_resource)?; + let next_configuration = configuration.bump_epoch_for_reconfiguration(); + let new_block_event = NewBlockEvent::new( + CORE_CODE_ADDRESS, + configuration.epoch(), + 0, + block_resource.height() + 1, + vec![], + CORE_CODE_ADDRESS, + vec![], + next_configuration.last_reconfiguration_time_micros(), + ); let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; let fork_txn = Transaction::GenesisTransaction(WriteSetPayload::Direct(ChangeSet::new( WriteSetMut::new(vec![ @@ -317,7 +354,7 @@ fn commit_fork_transaction( ( StateKey::on_chain_config::()?, WriteOp::legacy_modification( - bcs::to_bytes(&configuration.bump_epoch_for_reconfiguration())?.into(), + bcs::to_bytes(&next_configuration)?.into(), ), ), ]) @@ -326,9 +363,9 @@ fn commit_fork_transaction( ContractEvent::new_v2(NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG.clone(), vec![])?, ContractEvent::new_v1( new_block_event_key(), - 0, + block_resource.new_block_events().count(), TypeTag::Struct(Box::new(NewBlockEvent::struct_tag())), - vec![], + bcs::to_bytes(&new_block_event)?, )?, ], ))); @@ -421,7 +458,7 @@ fn write_manifest( "fork_validator_addresses": validator_addresses, }, "generated_config_paths": node_config_paths, - "waypoint": output_info.waypoint.to_string(), + "waypoint": output_info.waypoint.map(|waypoint| waypoint.to_string()), }); let manifest_path = config_dir.join(MANIFEST); fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; @@ -433,18 +470,31 @@ fn ledger_json(info: &DbInfo) -> serde_json::Value { "version": info.version, "epoch": info.epoch, "next_epoch": info.next_epoch, - "waypoint": info.waypoint.to_string(), + "waypoint": info.waypoint.map(|waypoint| waypoint.to_string()), "accumulator_root": info.accumulator_root, "state_root": info.state_root, "chain_id": info.chain_id.id(), + "new_block_event_count": info.new_block_event_count, }) } #[cfg(test)] mod tests { use super::*; - use aptos_executor_test_helpers::bootstrap_genesis; + use aptos_cached_packages::aptos_stdlib; + use aptos_crypto::PrivateKey; + use aptos_executor::block_executor::BlockExecutor; + use aptos_executor_test_helpers::{bootstrap_genesis, gen_block_id}; + use aptos_executor_types::BlockExecutorTrait; use aptos_temppath::TempPath; + use aptos_types::{ + aggregate_signature::AggregateSignature, + block_info::BlockInfo, + ledger_info::{LedgerInfo, LedgerInfoWithSignatures}, + test_helpers::transaction_test_helpers::{ + block, get_test_signed_transaction, TEST_BLOCK_EXECUTOR_ONCHAIN_CONFIG, + }, + }; #[test] fn rejects_reserved_chain_ids() { @@ -465,12 +515,38 @@ mod tests { } #[test] - fn fork_commits_chain_id_validator_set_and_waypoint_to_output_db() { + fn rejects_multi_validator_single_db_output() { + let root = TempPath::new(); + root.create_as_dir().unwrap(); + let source = root.path().join("source"); + let output = root.path().join("output"); + fs::create_dir_all(&source).unwrap(); + let error = Command { + source_db_dir: source, + output_db_dir: output, + config_dir: None, + fork_chain_id: 42, + validators: 2, + enable_storage_sharding: false, + } + .run() + .unwrap_err(); + assert!(error + .to_string() + .contains("--validators > 1 is not supported")); + } + + fn bootstrap_source_db() -> ( + TempPath, + TempPath, + aptos_crypto::ed25519::Ed25519PrivateKey, + Vec, + ) { let source_db_dir = TempPath::new(); source_db_dir.create_as_dir().unwrap(); let source_config_dir = TempPath::new(); source_config_dir.create_as_dir().unwrap(); - let (_root_key, genesis, _waypoint, _validators) = Builder::new( + let (root_key, genesis, _waypoint, validators) = Builder::new( source_config_dir.path(), aptos_cached_packages::head_release_bundle().clone(), ) @@ -484,7 +560,66 @@ mod tests { bootstrap_genesis::(&source_db, &genesis).unwrap(); } - let source_info_before = inspect_db(source_db_dir.path(), false).unwrap(); + (source_db_dir, source_config_dir, root_key, validators) + } + + fn commit_non_reconfiguration_transaction( + source_db_dir: &Path, + root_key: aptos_crypto::ed25519::Ed25519PrivateKey, + ) { + let db = open_db(source_db_dir, false, false).unwrap(); + let block_id = gen_block_id(7); + let txn = Transaction::UserTransaction(get_test_signed_transaction( + aptos_types::account_config::aptos_test_root_address(), + 0, + &root_key, + root_key.public_key(), + Some(aptos_stdlib::aptos_account_create_account( + AccountAddress::TWO, + )), + u64::MAX, + 0, + None, + )); + let executor = BlockExecutor::::new(db.clone()); + let output = executor + .execute_block( + (block_id, block(vec![txn])).into(), + executor.committed_block_id(), + TEST_BLOCK_EXECUTOR_ONCHAIN_CONFIG, + ) + .unwrap(); + let latest_li = db.reader.get_latest_ledger_info().unwrap(); + let ledger_info_with_sigs = LedgerInfoWithSignatures::new( + LedgerInfo::new( + BlockInfo::new( + latest_li.ledger_info().next_block_epoch(), + 0, + block_id, + output.root_hash(), + output.expect_last_version(), + 0, + None, + ), + gen_block_id(0), + ), + AggregateSignature::empty(), + ); + executor + .commit_blocks(vec![block_id], ledger_info_with_sigs) + .unwrap(); + assert!(inspect_db(source_db_dir, false, false) + .unwrap() + .waypoint + .is_none()); + } + + #[test] + fn fork_commits_chain_id_validator_set_and_waypoint_to_output_db() { + let (source_db_dir, _source_config_dir, root_key, _validators) = bootstrap_source_db(); + commit_non_reconfiguration_transaction(source_db_dir.path(), root_key); + + let source_info_before = inspect_db(source_db_dir.path(), false, false).unwrap(); let output_root = TempPath::new(); output_root.create_as_dir().unwrap(); let output_db_dir = output_root.path().join("fork-db"); @@ -502,8 +637,8 @@ mod tests { .run() .unwrap(); - let source_info_after = inspect_db(source_db_dir.path(), false).unwrap(); - let output_info = inspect_db(&output_db_dir, false).unwrap(); + let source_info_after = inspect_db(source_db_dir.path(), false, false).unwrap(); + let output_info = inspect_db(&output_db_dir, false, true).unwrap(); assert_eq!(source_info_after.chain_id, source_info_before.chain_id); assert_eq!( source_info_after.validator_set, @@ -513,8 +648,29 @@ mod tests { assert_ne!(output_info.chain_id, source_info_before.chain_id); assert_ne!(output_info.validator_set, source_info_before.validator_set); assert_eq!(output_info.validator_set.num_validators(), 1); - assert_eq!(output_info.waypoint.version(), source_info_before.version + 1); + assert_eq!( + output_info.waypoint.unwrap().version(), + source_info_before.version + 1 + ); assert!(output_info.next_epoch.is_some()); assert!(config_dir.join(MANIFEST).exists()); + + let genesis_txn = { + let bytes = fs::read(config_dir.join("0").join(GENESIS_BLOB)).unwrap(); + bcs::from_bytes::(&bytes).unwrap() + }; + let Transaction::GenesisTransaction(WriteSetPayload::Direct(change_set)) = genesis_txn else { + panic!("fork transaction must be a direct genesis write-set"); + }; + let new_block_event = change_set + .events() + .iter() + .find(|event| event.event_key() == Some(&new_block_event_key())) + .expect("fork transaction must emit NewBlockEvent"); + assert_eq!( + new_block_event.v1().unwrap().sequence_number(), + source_info_before.new_block_event_count + ); + NewBlockEvent::try_from_bytes(new_block_event.event_data()).unwrap(); } } From 1f6fd379c809449dd59d714813a315f93ec904bd Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Fri, 17 Jul 2026 10:54:09 -0700 Subject: [PATCH 03/21] Keep fork resources coherent --- Cargo.lock | 1 + storage/db-tool/Cargo.toml | 1 + storage/db-tool/src/fork.rs | 281 ++++++++++++++++++++++++++++++++---- 3 files changed, 254 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a7d569aabb..034913c06ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1216,6 +1216,7 @@ dependencies = [ "move-core-types", "rand 0.7.3", "rayon", + "serde", "serde_json", "serial_test", "tokio", diff --git a/storage/db-tool/Cargo.toml b/storage/db-tool/Cargo.toml index 80f361cdd19..d610ad64fa2 100644 --- a/storage/db-tool/Cargo.toml +++ b/storage/db-tool/Cargo.toml @@ -33,6 +33,7 @@ itertools = { workspace = true } move-core-types = { workspace = true } rand = { workspace = true } rayon = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index 612585ba5f2..5759d39a3f5 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -21,9 +21,14 @@ use aptos_types::{ }, chain_id::ChainId, contract_event::ContractEvent, - on_chain_config::{ConfigurationResource, OnChainConfig, ValidatorSet}, - state_store::{state_key::StateKey, TStateView}, + event::EventHandle, + on_chain_config::{ + CommitHistoryResource, ConfigurationResource, CurrentTimeMicroseconds, OnChainConfig, + ValidatorSet, + }, + state_store::{state_key::StateKey, table::TableHandle, TStateView}, transaction::{ChangeSet, Transaction, WriteSetPayload}, + validator_config::ValidatorConfig, waypoint::Waypoint, write_set::{WriteOp, WriteSetMut}, }; @@ -31,8 +36,10 @@ use aptos_vm::aptos_vm::AptosVMBlockExecutor; use clap::Parser; use move_core_types::{language_storage::TypeTag, move_resource::MoveStructType}; use rand::rngs::OsRng; +use serde::{Deserialize, Serialize}; use serde_json::json; use std::{ + collections::HashSet, fs::{self, File}, io::Write, num::NonZeroUsize, @@ -186,6 +193,31 @@ struct DbInfo { epoch: u64, next_epoch: Option, new_block_event_count: u64, + block_height: u64, + current_time_microseconds: u64, + commit_history_length: Option, + commit_history_next_idx: Option, +} + +#[derive(Deserialize, Serialize)] +struct BlockResourceWrite { + height: u64, + epoch_interval: u64, + new_block_events: EventHandle, + update_epoch_interval_events: EventHandle, +} + +#[derive(Deserialize, Serialize)] +struct TableWithLengthWrite { + handle: TableHandle, + length: u64, +} + +#[derive(Deserialize, Serialize)] +struct CommitHistoryResourceWrite { + max_capacity: u32, + next_idx: u32, + table: TableWithLengthWrite, } fn validate_requested_chain_id(chain_id: u8) -> Result { @@ -282,6 +314,9 @@ fn inspect_db(db_dir: &Path, sharding: bool, require_epoch_waypoint: bool) -> Re .get_state_value_bytes(&StateKey::resource_typed::(&CORE_CODE_ADDRESS)?)? .ok_or_else(|| format_err!("BlockResource missing"))?; let block_resource = bcs::from_bytes::(&block_resource)?; + let current_time = CurrentTimeMicroseconds::fetch_config(&view) + .ok_or_else(|| format_err!("CurrentTimeMicroseconds missing"))?; + let commit_history = CommitHistoryResource::fetch_config(&view); let ledger_info = db.reader.get_latest_ledger_info()?; let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; let waypoint = if ledger_info.ledger_info().ends_epoch() { @@ -310,6 +345,12 @@ fn inspect_db(db_dir: &Path, sharding: bool, require_epoch_waypoint: bool) -> Re .next_epoch_state() .map(|state| state.epoch), new_block_event_count: block_resource.new_block_events().count(), + block_height: block_resource.height(), + current_time_microseconds: current_time.microseconds, + commit_history_length: commit_history.as_ref().map(CommitHistoryResource::length), + commit_history_next_idx: commit_history + .as_ref() + .map(CommitHistoryResource::next_idx), }) } @@ -323,47 +364,63 @@ fn commit_fork_transaction( let view = db.reader.latest_state_checkpoint_view()?; let configuration = ConfigurationResource::fetch_config(&view) .ok_or_else(|| format_err!("ConfigurationResource missing"))?; + let source_validator_set = + ValidatorSet::fetch_config(&view).ok_or_else(|| format_err!("ValidatorSet missing"))?; + let current_time = CurrentTimeMicroseconds::fetch_config(&view) + .ok_or_else(|| format_err!("CurrentTimeMicroseconds missing"))?; + let block_resource_key = StateKey::resource_typed::(&CORE_CODE_ADDRESS)?; let block_resource = view - .get_state_value_bytes(&StateKey::resource_typed::(&CORE_CODE_ADDRESS)?)? + .get_state_value_bytes(&block_resource_key)? .ok_or_else(|| format_err!("BlockResource missing"))?; - let block_resource = bcs::from_bytes::(&block_resource)?; + let mut block_resource = bcs::from_bytes::(&block_resource)?; + let new_block_event_sequence = block_resource.new_block_events.count(); + block_resource.height = new_block_event_sequence; + *block_resource.new_block_events.count_mut() = new_block_event_sequence + 1; let next_configuration = configuration.bump_epoch_for_reconfiguration(); let new_block_event = NewBlockEvent::new( CORE_CODE_ADDRESS, configuration.epoch(), - 0, - block_resource.height() + 1, + u64::MAX, + new_block_event_sequence, vec![], - CORE_CODE_ADDRESS, + AccountAddress::ZERO, vec![], - next_configuration.last_reconfiguration_time_micros(), + current_time.microseconds, ); + let mut writes = vec![ + ( + StateKey::on_chain_config::()?, + WriteOp::legacy_modification( + bcs::to_bytes(&ChainIdResource::new(fork_chain_id))?.into(), + ), + ), + ( + StateKey::on_chain_config::()?, + WriteOp::legacy_modification(bcs::to_bytes(validator_set)?.into()), + ), + ( + StateKey::on_chain_config::()?, + WriteOp::legacy_modification(bcs::to_bytes(&next_configuration)?.into()), + ), + ( + block_resource_key, + WriteOp::legacy_modification(bcs::to_bytes(&block_resource)?.into()), + ), + ]; + writes.extend(commit_history_writes(&view, &new_block_event)?); + writes.extend(validator_config_writes( + &view, + &source_validator_set, + validator_set, + )?); let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; let fork_txn = Transaction::GenesisTransaction(WriteSetPayload::Direct(ChangeSet::new( - WriteSetMut::new(vec![ - ( - StateKey::on_chain_config::()?, - WriteOp::legacy_modification( - bcs::to_bytes(&ChainIdResource::new(fork_chain_id))?.into(), - ), - ), - ( - StateKey::on_chain_config::()?, - WriteOp::legacy_modification(bcs::to_bytes(validator_set)?.into()), - ), - ( - StateKey::on_chain_config::()?, - WriteOp::legacy_modification( - bcs::to_bytes(&next_configuration)?.into(), - ), - ), - ]) - .freeze()?, + WriteSetMut::new(writes).freeze()?, vec![ ContractEvent::new_v2(NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG.clone(), vec![])?, ContractEvent::new_v1( new_block_event_key(), - block_resource.new_block_events().count(), + new_block_event_sequence, TypeTag::Struct(Box::new(NewBlockEvent::struct_tag())), bcs::to_bytes(&new_block_event)?, )?, @@ -379,6 +436,84 @@ fn commit_fork_transaction( Ok((fork_txn, waypoint)) } +fn commit_history_writes( + view: &impl TStateView, + new_block_event: &NewBlockEvent, +) -> Result> { + let commit_history_key = StateKey::on_chain_config::()?; + let Some(commit_history) = view.get_state_value_bytes(&commit_history_key)? else { + return Ok(vec![]); + }; + let mut commit_history = bcs::from_bytes::(&commit_history)?; + ensure!( + commit_history.max_capacity > 0, + "CommitHistory max_capacity must be non-zero", + ); + let table_key = bcs::to_bytes(&commit_history.next_idx)?; + let table_state_key = StateKey::table_item(&commit_history.table.handle, &table_key); + let replacing_existing_slot = view.get_state_value_bytes(&table_state_key)?.is_some(); + let table_write_op = if replacing_existing_slot { + WriteOp::legacy_modification(bcs::to_bytes(new_block_event)?.into()) + } else { + commit_history.table.length += 1; + WriteOp::legacy_creation(bcs::to_bytes(new_block_event)?.into()) + }; + commit_history.next_idx = (commit_history.next_idx + 1) % commit_history.max_capacity; + Ok(vec![ + (table_state_key, table_write_op), + ( + commit_history_key, + WriteOp::legacy_modification(bcs::to_bytes(&commit_history)?.into()), + ), + ]) +} + +fn validator_config_writes( + view: &impl TStateView, + source_validator_set: &ValidatorSet, + validator_set: &ValidatorSet, +) -> Result> { + let new_addresses = validator_set + .active_validators + .iter() + .chain(validator_set.pending_inactive.iter()) + .chain(validator_set.pending_active.iter()) + .map(|validator| validator.account_address) + .collect::>(); + let mut writes = vec![]; + for validator in source_validator_set + .active_validators + .iter() + .chain(source_validator_set.pending_inactive.iter()) + .chain(source_validator_set.pending_active.iter()) + { + if new_addresses.contains(&validator.account_address) { + continue; + } + let key = StateKey::resource_typed::(&validator.account_address)?; + if view.get_state_value_bytes(&key)?.is_some() { + writes.push((key, WriteOp::legacy_deletion())); + } + } + + for validator in validator_set + .active_validators + .iter() + .chain(validator_set.pending_inactive.iter()) + .chain(validator_set.pending_active.iter()) + { + let key = StateKey::resource_typed::(&validator.account_address)?; + let bytes = bcs::to_bytes(validator.config())?.into(); + let write_op = if view.get_state_value_bytes(&key)?.is_some() { + WriteOp::legacy_modification(bytes) + } else { + WriteOp::legacy_creation(bytes) + }; + writes.push((key, write_op)); + } + Ok(writes) +} + fn open_db(db_dir: &Path, readonly: bool, sharding: bool) -> Result { let rocksdb_configs = RocksdbConfigs { enable_storage_sharding: sharding, @@ -475,6 +610,10 @@ fn ledger_json(info: &DbInfo) -> serde_json::Value { "state_root": info.state_root, "chain_id": info.chain_id.id(), "new_block_event_count": info.new_block_event_count, + "block_height": info.block_height, + "current_time_microseconds": info.current_time_microseconds, + "commit_history_length": info.commit_history_length, + "commit_history_next_idx": info.commit_history_next_idx, }) } @@ -614,12 +753,33 @@ mod tests { .is_none()); } + fn read_validator_config(db_dir: &Path, address: AccountAddress) -> Option { + let db = open_db(db_dir, true, false).unwrap(); + let view = db.reader.latest_state_checkpoint_view().unwrap(); + let bytes = view + .get_state_value_bytes(&StateKey::resource_typed::(&address).unwrap()) + .unwrap()?; + Some(bcs::from_bytes(&bytes).unwrap()) + } + #[test] fn fork_commits_chain_id_validator_set_and_waypoint_to_output_db() { let (source_db_dir, _source_config_dir, root_key, _validators) = bootstrap_source_db(); commit_non_reconfiguration_transaction(source_db_dir.path(), root_key); let source_info_before = inspect_db(source_db_dir.path(), false, false).unwrap(); + let source_validator = source_info_before + .validator_set + .active_validators + .first() + .unwrap(); + let source_validator_address = source_validator.account_address; + let source_validator_config = read_validator_config( + source_db_dir.path(), + source_validator_address, + ) + .expect("source validator config must exist before fork"); + assert_eq!(&source_validator_config, source_validator.config()); let output_root = TempPath::new(); output_root.create_as_dir().unwrap(); let output_db_dir = output_root.path().join("fork-db"); @@ -644,6 +804,10 @@ mod tests { source_info_after.validator_set, source_info_before.validator_set ); + assert_eq!( + read_validator_config(source_db_dir.path(), source_validator_address).unwrap(), + source_validator_config, + ); assert_eq!(output_info.chain_id, fork_chain_id); assert_ne!(output_info.chain_id, source_info_before.chain_id); assert_ne!(output_info.validator_set, source_info_before.validator_set); @@ -654,6 +818,58 @@ mod tests { ); assert!(output_info.next_epoch.is_some()); assert!(config_dir.join(MANIFEST).exists()); + assert_eq!( + output_info.new_block_event_count, + source_info_before.new_block_event_count + 1 + ); + assert_eq!( + output_info.block_height, + source_info_before.new_block_event_count + ); + assert_eq!( + output_info.current_time_microseconds, + source_info_before.current_time_microseconds + ); + assert_eq!( + output_info.commit_history_length, + source_info_before + .commit_history_length + .map(|length| length + 1) + ); + assert_eq!( + output_info.commit_history_next_idx, + source_info_before + .commit_history_next_idx + .map(|next_idx| next_idx + 1) + ); + + let output_validator = output_info + .validator_set + .active_validators + .first() + .unwrap(); + let output_validator_address = output_validator.account_address; + assert_ne!(output_validator_address, source_validator_address); + assert!( + read_validator_config(&output_db_dir, source_validator_address).is_none(), + "old validator config must not remain in fork output", + ); + let output_validator_config = + read_validator_config(&output_db_dir, output_validator_address) + .expect("generated validator config must exist in fork output"); + assert_eq!(&output_validator_config, output_validator.config()); + assert_ne!( + output_validator_config.validator_network_addresses, + source_validator_config.validator_network_addresses, + ); + assert_ne!( + output_validator_config.fullnode_network_addresses, + source_validator_config.fullnode_network_addresses, + ); + assert_ne!( + output_validator_config.consensus_public_key, + source_validator_config.consensus_public_key, + ); let genesis_txn = { let bytes = fs::read(config_dir.join("0").join(GENESIS_BLOB)).unwrap(); @@ -671,6 +887,13 @@ mod tests { new_block_event.v1().unwrap().sequence_number(), source_info_before.new_block_event_count ); - NewBlockEvent::try_from_bytes(new_block_event.event_data()).unwrap(); + let new_block_event = NewBlockEvent::try_from_bytes(new_block_event.event_data()).unwrap(); + assert_eq!(new_block_event.height(), source_info_before.new_block_event_count); + assert_eq!(new_block_event.round(), u64::MAX); + assert_eq!(new_block_event.proposer(), AccountAddress::ZERO); + assert_eq!( + new_block_event.proposed_time(), + source_info_before.current_time_microseconds, + ); } } From cf6c5b1ec140e0a57be5ea9abf3adcc1d59cbe89 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Fri, 17 Jul 2026 21:51:56 -0700 Subject: [PATCH 04/21] Prove fork preserves post-genesis state --- storage/db-tool/src/fork.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index 5759d39a3f5..89dc94c5778 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -629,6 +629,7 @@ mod tests { use aptos_types::{ aggregate_signature::AggregateSignature, block_info::BlockInfo, + account_config::AccountResource, ledger_info::{LedgerInfo, LedgerInfoWithSignatures}, test_helpers::transaction_test_helpers::{ block, get_test_signed_transaction, TEST_BLOCK_EXECUTOR_ONCHAIN_CONFIG, @@ -762,12 +763,28 @@ mod tests { Some(bcs::from_bytes(&bytes).unwrap()) } + fn read_account_resource(db_dir: &Path, address: AccountAddress) -> Option { + let db = open_db(db_dir, true, false).unwrap(); + let view = db.reader.latest_state_checkpoint_view().unwrap(); + let bytes = view + .get_state_value_bytes(&StateKey::resource_typed::(&address).unwrap()) + .unwrap()?; + Some(bcs::from_bytes(&bytes).unwrap()) + } + #[test] fn fork_commits_chain_id_validator_set_and_waypoint_to_output_db() { let (source_db_dir, _source_config_dir, root_key, _validators) = bootstrap_source_db(); commit_non_reconfiguration_transaction(source_db_dir.path(), root_key); let source_info_before = inspect_db(source_db_dir.path(), false, false).unwrap(); + assert!( + source_info_before.version > 0, + "source DB must have committed post-genesis transactions before fork", + ); + let source_account = read_account_resource(source_db_dir.path(), AccountAddress::TWO) + .expect("0x2::account::Account must exist before fork"); + assert_eq!(source_account.sequence_number(), 0); let source_validator = source_info_before .validator_set .active_validators @@ -808,6 +825,10 @@ mod tests { read_validator_config(source_db_dir.path(), source_validator_address).unwrap(), source_validator_config, ); + assert_eq!( + read_account_resource(source_db_dir.path(), AccountAddress::TWO).unwrap(), + source_account, + ); assert_eq!(output_info.chain_id, fork_chain_id); assert_ne!(output_info.chain_id, source_info_before.chain_id); assert_ne!(output_info.validator_set, source_info_before.validator_set); @@ -842,6 +863,10 @@ mod tests { .commit_history_next_idx .map(|next_idx| next_idx + 1) ); + assert_eq!( + read_account_resource(&output_db_dir, AccountAddress::TWO).unwrap(), + source_account, + ); let output_validator = output_info .validator_set From c9bafd00226cdcc66b191fd2b2b1b29fa035ff2a Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 20 Jul 2026 11:07:48 -0700 Subject: [PATCH 05/21] Support two-validator fork bootstrap --- Cargo.lock | 1 + storage/db-tool/Cargo.toml | 1 + storage/db-tool/src/fork.rs | 293 ++++++++++++++++++++++++++++++++---- 3 files changed, 264 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 034913c06ac..631e6207182 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1216,6 +1216,7 @@ dependencies = [ "move-core-types", "rand 0.7.3", "rayon", + "reqwest 0.11.23", "serde", "serde_json", "serial_test", diff --git a/storage/db-tool/Cargo.toml b/storage/db-tool/Cargo.toml index d610ad64fa2..9fb70851c8b 100644 --- a/storage/db-tool/Cargo.toml +++ b/storage/db-tool/Cargo.toml @@ -43,4 +43,5 @@ aptos-backup-service = { workspace = true } aptos-crypto = { workspace = true } aptos-executor-test-helpers = { workspace = true } aptos-indexer-grpc-table-info = { workspace = true } +reqwest = { workspace = true } serial_test = { workspace = true } diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index 89dc94c5778..23cfdff9951 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -29,12 +29,15 @@ use aptos_types::{ state_store::{state_key::StateKey, table::TableHandle, TStateView}, transaction::{ChangeSet, Transaction, WriteSetPayload}, validator_config::ValidatorConfig, + validator_performances::{ValidatorPerformance, ValidatorPerformances}, waypoint::Waypoint, write_set::{WriteOp, WriteSetMut}, }; use aptos_vm::aptos_vm::AptosVMBlockExecutor; use clap::Parser; -use move_core_types::{language_storage::TypeTag, move_resource::MoveStructType}; +use move_core_types::{ + language_storage::TypeTag, move_resource::MoveStructType, parser::parse_struct_tag, +}; use rand::rngs::OsRng; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -79,10 +82,6 @@ impl Command { pub fn run(self) -> Result<()> { let validators = NonZeroUsize::new(self.validators) .ok_or_else(|| format_err!("--validators must be greater than 0"))?; - ensure!( - validators.get() == 1, - "--validators > 1 is not supported until per-validator DB output is implemented", - ); let fork_chain_id = validate_requested_chain_id(self.fork_chain_id)?; validate_output_paths(&self.source_db_dir, &self.output_db_dir)?; let config_dir = self @@ -115,7 +114,7 @@ impl Command { fs::create_dir_all(&config_dir) .with_context(|| format!("failed to create config dir {:?}", config_dir))?; - let (_root_key, _generated_genesis, _generated_waypoint, validators) = Builder::new( + let (_root_key, generated_genesis, _generated_waypoint, validators) = Builder::new( &config_dir, aptos_cached_packages::head_release_bundle().clone(), )? @@ -129,6 +128,7 @@ impl Command { self.enable_storage_sharding, fork_chain_id, &validator_set, + &generated_genesis, )?; let output_info = inspect_db(&self.output_db_dir, self.enable_storage_sharding, true) @@ -163,6 +163,7 @@ impl Command { &fork_txn, waypoint, fork_chain_id, + self.enable_storage_sharding, )?; let manifest_path = write_manifest( &config_dir, @@ -359,6 +360,7 @@ fn commit_fork_transaction( sharding: bool, fork_chain_id: ChainId, validator_set: &ValidatorSet, + generated_genesis: &Transaction, ) -> Result<(Transaction, Waypoint)> { let db = open_db(output_db_dir, false, sharding)?; let view = db.reader.latest_state_checkpoint_view()?; @@ -413,6 +415,11 @@ fn commit_fork_transaction( &source_validator_set, validator_set, )?); + writes.extend(validator_stake_writes( + &view, + validator_set, + generated_genesis, + )?); let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; let fork_txn = Transaction::GenesisTransaction(WriteSetPayload::Direct(ChangeSet::new( WriteSetMut::new(writes).freeze()?, @@ -436,6 +443,60 @@ fn commit_fork_transaction( Ok((fork_txn, waypoint)) } +fn validator_stake_writes( + view: &impl TStateView, + validator_set: &ValidatorSet, + generated_genesis: &Transaction, +) -> Result> { + let Transaction::GenesisTransaction(WriteSetPayload::Direct(generated_change_set)) = + generated_genesis + else { + return Err(format_err!( + "generated validator genesis must be a direct write-set" + )); + }; + let stake_pool_tag = parse_struct_tag("0x1::stake::StakePool")?; + let mut writes = Vec::with_capacity(validator_set.num_validators() + 1); + for validator in validator_set.active_validators() { + let stake_pool_key = StateKey::resource(&validator, &stake_pool_tag)?; + ensure!( + view.get_state_value_bytes(&stake_pool_key)?.is_none(), + "generated validator address {} already has a StakePool in source state", + validator, + ); + let stake_pool_write = generated_change_set + .write_set() + .get_write_op(&stake_pool_key) + .cloned() + .ok_or_else(|| { + format_err!( + "generated genesis is missing StakePool for validator {}", + validator + ) + })?; + writes.push((stake_pool_key, stake_pool_write)); + } + + let validator_performance_key = StateKey::resource( + &CORE_CODE_ADDRESS, + &parse_struct_tag("0x1::stake::ValidatorPerformance")?, + )?; + let validator_performances = ValidatorPerformances { + validators: vec![ + ValidatorPerformance { + successful_proposals: 0, + failed_proposals: 0, + }; + validator_set.num_validators() + ], + }; + writes.push(( + validator_performance_key, + WriteOp::legacy_modification(bcs::to_bytes(&validator_performances)?.into()), + )); + Ok(writes) +} + fn commit_history_writes( view: &impl TStateView, new_block_event: &NewBlockEvent, @@ -538,13 +599,32 @@ fn write_fork_configs( fork_txn: &Transaction, waypoint: Waypoint, fork_chain_id: ChainId, + enable_storage_sharding: bool, ) -> Result> { validators .iter() .map(|validator| { let config_path = validator.validator_config_path(); let mut config = NodeConfig::load_from_path(&config_path)?; - config.storage.dir = output_db_dir.to_path_buf(); + let validator_db_dir = validator.dir.join("fork-db"); + fs::create_dir_all(&validator_db_dir).with_context(|| { + format!( + "failed to create validator {} DB directory at {:?}", + validator.index, validator_db_dir + ) + })?; + AptosDB::create_checkpoint( + output_db_dir, + &validator_db_dir, + enable_storage_sharding, + ) + .with_context(|| { + format!( + "failed to create validator {} DB checkpoint at {:?}", + validator.index, validator_db_dir + ) + })?; + config.storage.dir = validator_db_dir; config.base.waypoint = WaypointConfig::FromConfig(waypoint); config.execution.genesis = Some(fork_txn.clone()); config.execution.genesis_waypoint = Some(WaypointConfig::FromConfig(waypoint)); @@ -635,6 +715,21 @@ mod tests { block, get_test_signed_transaction, TEST_BLOCK_EXECUTOR_ONCHAIN_CONFIG, }, }; + use std::{ + process::{Child, Command as ProcessCommand, Stdio}, + time::Duration, + }; + + struct NodeProcesses(Vec); + + impl Drop for NodeProcesses { + fn drop(&mut self) { + for child in &mut self.0 { + let _ = child.kill(); + let _ = child.wait(); + } + } + } #[test] fn rejects_reserved_chain_ids() { @@ -654,28 +749,6 @@ mod tests { assert!(validate_output_paths(&source, &root.path().join("output")).is_ok()); } - #[test] - fn rejects_multi_validator_single_db_output() { - let root = TempPath::new(); - root.create_as_dir().unwrap(); - let source = root.path().join("source"); - let output = root.path().join("output"); - fs::create_dir_all(&source).unwrap(); - let error = Command { - source_db_dir: source, - output_db_dir: output, - config_dir: None, - fork_chain_id: 42, - validators: 2, - enable_storage_sharding: false, - } - .run() - .unwrap_err(); - assert!(error - .to_string() - .contains("--validators > 1 is not supported")); - } - fn bootstrap_source_db() -> ( TempPath, TempPath, @@ -772,6 +845,16 @@ mod tests { Some(bcs::from_bytes(&bytes).unwrap()) } + fn resource_exists(db_dir: &Path, address: AccountAddress, type_name: &str) -> bool { + let db = open_db(db_dir, true, false).unwrap(); + let view = db.reader.latest_state_checkpoint_view().unwrap(); + view.get_state_value_bytes( + &StateKey::resource(&address, &parse_struct_tag(type_name).unwrap()).unwrap(), + ) + .unwrap() + .is_some() + } + #[test] fn fork_commits_chain_id_validator_set_and_waypoint_to_output_db() { let (source_db_dir, _source_config_dir, root_key, _validators) = bootstrap_source_db(); @@ -808,7 +891,7 @@ mod tests { output_db_dir: output_db_dir.clone(), config_dir: Some(config_dir.clone()), fork_chain_id: fork_chain_id.id(), - validators: 1, + validators: 2, enable_storage_sharding: false, } .run() @@ -832,7 +915,13 @@ mod tests { assert_eq!(output_info.chain_id, fork_chain_id); assert_ne!(output_info.chain_id, source_info_before.chain_id); assert_ne!(output_info.validator_set, source_info_before.validator_set); - assert_eq!(output_info.validator_set.num_validators(), 1); + assert_eq!(output_info.validator_set.num_validators(), 2); + for validator in output_info.validator_set.active_validators() { + assert!( + resource_exists(&output_db_dir, validator, "0x1::stake::StakePool"), + "replacement validator must have a StakePool", + ); + } assert_eq!( output_info.waypoint.unwrap().version(), source_info_before.version + 1 @@ -867,6 +956,28 @@ mod tests { read_account_resource(&output_db_dir, AccountAddress::TWO).unwrap(), source_account, ); + for validator_index in 0..2 { + let validator_db_dir = config_dir + .join(validator_index.to_string()) + .join("fork-db"); + let validator_info = inspect_db(&validator_db_dir, false, true).unwrap(); + assert_eq!(validator_info.chain_id, fork_chain_id); + assert_eq!(validator_info.validator_set, output_info.validator_set); + assert_eq!( + read_account_resource(&validator_db_dir, AccountAddress::TWO).unwrap(), + source_account, + ); + let node_config = NodeConfig::load_from_path( + &config_dir + .join(validator_index.to_string()) + .join("node.yaml"), + ) + .unwrap(); + assert_eq!( + node_config.storage.dir().canonicalize().unwrap(), + validator_db_dir.canonicalize().unwrap(), + ); + } let output_validator = output_info .validator_set @@ -921,4 +1032,124 @@ mod tests { source_info_before.current_time_microseconds, ); } + + #[tokio::test] + #[ignore = "requires APTOS_NODE_BINARY pointing to an aptos-node binary"] + async fn two_validator_fork_bootstraps_and_commits_blocks() { + let node_binary = PathBuf::from( + std::env::var_os("APTOS_NODE_BINARY") + .expect("APTOS_NODE_BINARY must point to an aptos-node binary"), + ); + let node_binary = if node_binary.is_absolute() { + node_binary + } else { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(node_binary) + } + .canonicalize() + .expect("APTOS_NODE_BINARY must resolve to an aptos-node binary"); + let (source_db_dir, _source_config_dir, root_key, _validators) = bootstrap_source_db(); + commit_non_reconfiguration_transaction(source_db_dir.path(), root_key); + let source_account = read_account_resource(source_db_dir.path(), AccountAddress::TWO) + .expect("post-genesis account must exist"); + + let output_root = TempPath::new(); + output_root.create_as_dir().unwrap(); + let output_db_dir = output_root.path().join("fork-db"); + let config_dir = output_root.path().join("fork-configs"); + Command { + source_db_dir: source_db_dir.path().to_path_buf(), + output_db_dir, + config_dir: Some(config_dir.clone()), + fork_chain_id: 42, + validators: 2, + enable_storage_sharding: false, + } + .run() + .unwrap(); + + let mut api_addresses = Vec::new(); + let mut children = Vec::new(); + for validator_index in 0..2 { + let validator_dir = config_dir.join(validator_index.to_string()); + let config_path = validator_dir.join("node.yaml"); + let config = NodeConfig::load_from_path(&config_path).unwrap(); + api_addresses.push(config.api.address); + let log = File::create(validator_dir.join("live-smoke.log")).unwrap(); + children.push( + ProcessCommand::new(&node_binary) + .current_dir(&validator_dir) + .arg("-f") + .arg(config_path) + .stdout(Stdio::from(log.try_clone().unwrap())) + .stderr(Stdio::from(log)) + .spawn() + .unwrap(), + ); + } + let mut nodes = NodeProcesses(children); + let client = reqwest::Client::new(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + let mut starting_versions = None; + while tokio::time::Instant::now() < deadline { + for child in &mut nodes.0 { + assert!(child.try_wait().unwrap().is_none(), "validator exited early"); + } + let mut ledger_infos = Vec::new(); + for address in &api_addresses { + if let Ok(response) = client + .get(format!("http://{}/v1/", address)) + .send() + .await + { + if let Ok(info) = response.json::().await { + ledger_infos.push(info); + } + } + } + if ledger_infos.len() == 2 { + assert!(ledger_infos.iter().all(|info| info["chain_id"] == 42)); + let versions = ledger_infos + .iter() + .map(|info| { + info["ledger_version"] + .as_str() + .unwrap() + .parse::() + .unwrap() + }) + .collect::>(); + let initial_versions = starting_versions.get_or_insert_with(|| versions.clone()); + if versions + .iter() + .zip(initial_versions.iter()) + .all(|(current, initial)| current > initial) + { + for address in &api_addresses { + let account = client + .get(format!( + "http://{}/v1/accounts/0x2/resource/0x1::account::Account", + address + )) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json::() + .await + .unwrap(); + assert_eq!( + account["data"]["sequence_number"], + source_account.sequence_number().to_string(), + ); + } + return; + } + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + panic!("two-validator fork did not commit a block before the deadline"); + } } From 7a84158e762fd353027739b0ef285f89c20bdaad Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 22 Jul 2026 16:47:37 -0700 Subject: [PATCH 06/21] Keep fork validators active across epochs --- storage/db-tool/src/fork.rs | 109 +++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 9 deletions(-) diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index 23cfdff9951..7f3bd4ab523 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -26,6 +26,7 @@ use aptos_types::{ CommitHistoryResource, ConfigurationResource, CurrentTimeMicroseconds, OnChainConfig, ValidatorSet, }, + network_address::{NetworkAddress, Protocol}, state_store::{state_key::StateKey, table::TableHandle, TStateView}, transaction::{ChangeSet, Transaction, WriteSetPayload}, validator_config::ValidatorConfig, @@ -45,8 +46,10 @@ use std::{ collections::HashSet, fs::{self, File}, io::Write, + net::{IpAddr, Ipv4Addr}, num::NonZeroUsize, path::{Path, PathBuf}, + sync::Arc, }; const GENESIS_BLOB: &str = "genesis.blob"; @@ -114,11 +117,21 @@ impl Command { fs::create_dir_all(&config_dir) .with_context(|| format!("failed to create config dir {:?}", config_dir))?; + let replacement_stake = source_info + .validator_set + .active_validators + .iter() + .map(|validator| validator.consensus_voting_power()) + .max() + .ok_or_else(|| format_err!("source ValidatorSet has no active validators"))?; let (_root_key, generated_genesis, _generated_waypoint, validators) = Builder::new( &config_dir, aptos_cached_packages::head_release_bundle().clone(), )? .with_num_validators(validators) + .with_init_genesis_stake(Some(Arc::new(move |_, stake| { + *stake = replacement_stake; + }))) .build(OsRng)?; let validator_set = validator_set_from_local_validators(&validators)?; let generated_validator_addresses = validator_set.active_validators(); @@ -174,6 +187,7 @@ impl Command { &output_info, fork_chain_id, &generated_validator_addresses, + replacement_stake, )?; println!("Fork DB written to: {}", self.output_db_dir.display()); @@ -304,7 +318,10 @@ fn default_config_dir(output_db_dir: &Path) -> PathBuf { } fn inspect_db(db_dir: &Path, sharding: bool, require_epoch_waypoint: bool) -> Result { - let db = open_db(db_dir, true, sharding)?; + // A read-only open can expose resource values from the last materialized state snapshot + // while ledger metadata is newer. The source is already required to be a disposable, + // writable checkpoint, so recover it before deriving fork parameters. + let db = open_db(db_dir, false, sharding)?; let view = db.reader.latest_state_checkpoint_view()?; let chain_id = ChainIdResource::fetch_config(&view) .ok_or_else(|| format_err!("ChainIdResource missing"))? @@ -635,6 +652,7 @@ fn write_fork_configs( vec![], WaypointConfig::FromConfig(waypoint), ); + isolate_node_config(&mut config)?; File::create(&config.execution.genesis_file_location)? .write_all(&bcs::to_bytes(fork_txn)?)?; config.save_to_path(&config_path)?; @@ -647,6 +665,38 @@ fn write_fork_configs( .collect() } +fn isolate_node_config(config: &mut NodeConfig) -> Result<()> { + let loopback = IpAddr::V4(Ipv4Addr::LOCALHOST); + config.api.address.set_ip(loopback); + config.indexer_grpc.address.set_ip(loopback); + config.admin_service.address = Ipv4Addr::LOCALHOST.to_string(); + config.inspection_service.address = Ipv4Addr::LOCALHOST.to_string(); + config.logger.enable_telemetry_remote_log = false; + config.logger.enable_telemetry_flush = false; + + for network in config + .validator_network + .iter_mut() + .chain(config.full_node_networks.iter_mut()) + { + let protocols = network + .listen_address + .as_slice() + .iter() + .map(|protocol| match protocol { + Protocol::Ip4(_) | Protocol::Ip6(_) | Protocol::Dns(_) + | Protocol::Dns4(_) | Protocol::Dns6(_) => { + Protocol::Ip4(Ipv4Addr::LOCALHOST) + }, + protocol => protocol.clone(), + }) + .collect(); + network.listen_address = NetworkAddress::from_protocols(protocols) + .context("failed to bind generated network config to loopback")?; + } + Ok(()) +} + fn write_manifest( config_dir: &Path, source_db_dir: &Path, @@ -656,6 +706,7 @@ fn write_manifest( output_info: &DbInfo, fork_chain_id: ChainId, validator_addresses: &[AccountAddress], + replacement_stake: u64, ) -> Result { let manifest = json!({ "source_db_dir": source_db_dir, @@ -671,6 +722,7 @@ fn write_manifest( "fork_validator_count": output_info.validator_set.num_validators(), "source_validator_addresses": source_info.validator_set.active_validators(), "fork_validator_addresses": validator_addresses, + "replacement_stake": replacement_stake, }, "generated_config_paths": node_config_paths, "waypoint": output_info.waypoint.map(|waypoint| waypoint.to_string()), @@ -765,6 +817,11 @@ mod tests { ) .unwrap() .with_num_validators(NonZeroUsize::new(1).unwrap()) + .with_init_genesis_stake(Some(Arc::new(|_, stake| *stake = 1_000))) + .with_init_genesis_config(Some(Arc::new(|config| { + config.min_stake = 100; + config.max_stake = 10_000; + }))) .build(OsRng) .unwrap(); @@ -821,6 +878,8 @@ mod tests { executor .commit_blocks(vec![block_id], ledger_info_with_sigs) .unwrap(); + drop(executor); + drop(db); assert!(inspect_db(source_db_dir, false, false) .unwrap() .waypoint @@ -922,6 +981,11 @@ mod tests { "replacement validator must have a StakePool", ); } + assert!(output_info + .validator_set + .active_validators + .iter() + .all(|validator| validator.consensus_voting_power() == 1_000)); assert_eq!( output_info.waypoint.unwrap().version(), source_info_before.version + 1 @@ -977,6 +1041,18 @@ mod tests { node_config.storage.dir().canonicalize().unwrap(), validator_db_dir.canonicalize().unwrap(), ); + assert!(node_config.api.address.ip().is_loopback()); + assert!(node_config.indexer_grpc.address.ip().is_loopback()); + assert!(!node_config.logger.enable_telemetry_remote_log); + assert!(!node_config.logger.enable_telemetry_flush); + assert!(node_config + .validator_network + .iter() + .chain(node_config.full_node_networks.iter()) + .all(|network| matches!( + network.listen_address.as_slice().first(), + Some(Protocol::Ip4(address)) if address.is_loopback() + ))); } let output_validator = output_info @@ -1060,7 +1136,7 @@ mod tests { let config_dir = output_root.path().join("fork-configs"); Command { source_db_dir: source_db_dir.path().to_path_buf(), - output_db_dir, + output_db_dir: output_db_dir.clone(), config_dir: Some(config_dir.clone()), fork_chain_id: 42, validators: 2, @@ -1068,6 +1144,7 @@ mod tests { } .run() .unwrap(); + let fork_version = inspect_db(&output_db_dir, false, true).unwrap().version; let mut api_addresses = Vec::new(); let mut children = Vec::new(); @@ -1091,7 +1168,6 @@ mod tests { let mut nodes = NodeProcesses(children); let client = reqwest::Client::new(); let deadline = tokio::time::Instant::now() + Duration::from_secs(60); - let mut starting_versions = None; while tokio::time::Instant::now() < deadline { for child in &mut nodes.0 { assert!(child.try_wait().unwrap().is_none(), "validator exited early"); @@ -1120,12 +1196,7 @@ mod tests { .unwrap() }) .collect::>(); - let initial_versions = starting_versions.get_or_insert_with(|| versions.clone()); - if versions - .iter() - .zip(initial_versions.iter()) - .all(|(current, initial)| current > initial) - { + if versions.iter().all(|current| *current >= fork_version + 2) { for address in &api_addresses { let account = client .get(format!( @@ -1144,6 +1215,26 @@ mod tests { account["data"]["sequence_number"], source_account.sequence_number().to_string(), ); + let validator_set = client + .get(format!( + "http://{}/v1/accounts/0x1/resource/0x1::stake::ValidatorSet", + address + )) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json::() + .await + .unwrap(); + assert_eq!( + validator_set["data"]["active_validators"] + .as_array() + .unwrap() + .len(), + 2, + ); } return; } From 366cb598900c5a3dda9a4e21cb5d46dd678225d7 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 22 Jul 2026 21:55:01 -0700 Subject: [PATCH 07/21] Add funded test account rekeying --- Cargo.lock | 1 + storage/db-tool/Cargo.toml | 3 +- storage/db-tool/src/fork.rs | 190 +++++++++++++++++++++++++++++++++++- 3 files changed, 189 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 631e6207182..c77d77bc5d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1212,6 +1212,7 @@ dependencies = [ "aptos-vm", "bcs 0.1.4", "clap 4.5.21", + "hex", "itertools 0.13.0", "move-core-types", "rand 0.7.3", diff --git a/storage/db-tool/Cargo.toml b/storage/db-tool/Cargo.toml index 9fb70851c8b..f52807d3b03 100644 --- a/storage/db-tool/Cargo.toml +++ b/storage/db-tool/Cargo.toml @@ -17,6 +17,7 @@ aptos-backup-cli = { workspace = true } aptos-block-executor = { workspace = true } aptos-cached-packages = { workspace = true } aptos-config = { workspace = true } +aptos-crypto = { workspace = true } aptos-db = { workspace = true, features = ["db-debugger"] } aptos-db-indexer = { workspace = true } aptos-executor = { workspace = true } @@ -29,6 +30,7 @@ aptos-types = { workspace = true } aptos-vm = { workspace = true } bcs = { workspace = true } clap = { workspace = true } +hex = { workspace = true } itertools = { workspace = true } move-core-types = { workspace = true } rand = { workspace = true } @@ -40,7 +42,6 @@ tokio = { workspace = true } [dev-dependencies] aptos-backup-cli = { workspace = true, features = ["testing"] } aptos-backup-service = { workspace = true } -aptos-crypto = { workspace = true } aptos-executor-test-helpers = { workspace = true } aptos-indexer-grpc-table-info = { workspace = true } reqwest = { workspace = true } diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index 7f3bd4ab523..d0dd7d3bd0f 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -7,6 +7,7 @@ use aptos_config::config::{ BUFFERED_STATE_TARGET_ITEMS, DEFAULT_MAX_NUM_NODES_PER_LRU_CACHE_SHARD, NO_OP_STORAGE_PRUNER_CONFIG, }; +use aptos_crypto::{ed25519::Ed25519PrivateKey, PrivateKey}; use aptos_db::AptosDB; use aptos_executor::db_bootstrapper::{calculate_genesis, generate_waypoint}; use aptos_genesis::builder::{validator_set_from_local_validators, Builder, ValidatorNodeConfig}; @@ -16,8 +17,8 @@ use aptos_storage_interface::{ use aptos_types::{ account_address::AccountAddress, account_config::{ - new_block_event_key, BlockResource, ChainIdResource, NewBlockEvent, CORE_CODE_ADDRESS, - NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG, + new_block_event_key, AccountResource, BlockResource, ChainIdResource, NewBlockEvent, + CORE_CODE_ADDRESS, NEW_EPOCH_EVENT_V2_MOVE_TYPE_TAG, }, chain_id::ChainId, contract_event::ContractEvent, @@ -28,7 +29,9 @@ use aptos_types::{ }, network_address::{NetworkAddress, Protocol}, state_store::{state_key::StateKey, table::TableHandle, TStateView}, - transaction::{ChangeSet, Transaction, WriteSetPayload}, + transaction::{ + authenticator::AuthenticationKey, ChangeSet, Transaction, WriteSetPayload, + }, validator_config::ValidatorConfig, validator_performances::{ValidatorPerformance, ValidatorPerformances}, waypoint::Waypoint, @@ -44,17 +47,21 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use std::{ collections::HashSet, - fs::{self, File}, + fs::{self, File, OpenOptions}, io::Write, net::{IpAddr, Ipv4Addr}, num::NonZeroUsize, path::{Path, PathBuf}, sync::Arc, }; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; const GENESIS_BLOB: &str = "genesis.blob"; const MANIFEST: &str = "fork-manifest.json"; const VALIDATOR_IDENTITY: &str = "validator-identity.yaml"; +const TEST_ACCOUNT_ADDRESS: &str = "test-account-address"; +const TEST_ACCOUNT_PRIVATE_KEY: &str = "test-account-private-key"; #[derive(Parser)] #[clap( @@ -79,6 +86,12 @@ pub struct Command { #[clap(long)] enable_storage_sharding: bool, + + #[clap(long, requires = "test_account_private_key")] + test_account_address: Option, + + #[clap(long, requires = "test_account_address")] + test_account_private_key: Option, } impl Command { @@ -86,6 +99,10 @@ impl Command { let validators = NonZeroUsize::new(self.validators) .ok_or_else(|| format_err!("--validators must be greater than 0"))?; let fork_chain_id = validate_requested_chain_id(self.fork_chain_id)?; + let test_account_rekey = parse_test_account_rekey( + self.test_account_address, + self.test_account_private_key, + )?; validate_output_paths(&self.source_db_dir, &self.output_db_dir)?; let config_dir = self .config_dir @@ -142,6 +159,7 @@ impl Command { fork_chain_id, &validator_set, &generated_genesis, + test_account_rekey.as_ref(), )?; let output_info = inspect_db(&self.output_db_dir, self.enable_storage_sharding, true) @@ -178,6 +196,9 @@ impl Command { fork_chain_id, self.enable_storage_sharding, )?; + if let Some(test_account_rekey) = &test_account_rekey { + write_test_account_key(&config_dir, test_account_rekey)?; + } let manifest_path = write_manifest( &config_dir, &self.source_db_dir, @@ -188,6 +209,7 @@ impl Command { fork_chain_id, &generated_validator_addresses, replacement_stake, + test_account_rekey.as_ref(), )?; println!("Fork DB written to: {}", self.output_db_dir.display()); @@ -235,6 +257,54 @@ struct CommitHistoryResourceWrite { table: TableWithLengthWrite, } +#[derive(Deserialize, Serialize)] +struct AccountResourceWrite { + authentication_key: Vec, + sequence_number: u64, + guid_creation_num: u64, + coin_register_events: EventHandle, + key_rotation_events: EventHandle, + rotation_capability_offer: Option, + signer_capability_offer: Option, +} + +struct TestAccountRekey { + address: AccountAddress, + private_key_hex: String, + authentication_key: Vec, +} + +fn parse_test_account_rekey( + address: Option, + private_key: Option, +) -> Result> { + let (address, private_key) = match (address, private_key) { + (None, None) => return Ok(None), + (Some(address), Some(private_key)) => (address, private_key), + _ => { + return Err(format_err!( + "--test-account-address and --test-account-private-key must be provided together" + )); + }, + }; + let private_key = private_key.strip_prefix("0x").unwrap_or(&private_key); + ensure!( + !private_key.is_empty() && private_key.len() <= 64, + "test account private key must contain 1 to 64 hexadecimal digits", + ); + let private_key_hex = format!("{:0>64}", private_key); + let private_key_bytes = + hex::decode(&private_key_hex).context("test account private key is not valid hex")?; + let private_key = Ed25519PrivateKey::try_from(private_key_bytes.as_slice()) + .context("test account private key is not a valid Ed25519 private key")?; + let authentication_key = AuthenticationKey::ed25519(&private_key.public_key()).to_vec(); + Ok(Some(TestAccountRekey { + address, + private_key_hex, + authentication_key, + })) +} + fn validate_requested_chain_id(chain_id: u8) -> Result { ensure!( chain_id != 0 && chain_id != 1, @@ -378,6 +448,7 @@ fn commit_fork_transaction( fork_chain_id: ChainId, validator_set: &ValidatorSet, generated_genesis: &Transaction, + test_account_rekey: Option<&TestAccountRekey>, ) -> Result<(Transaction, Waypoint)> { let db = open_db(output_db_dir, false, sharding)?; let view = db.reader.latest_state_checkpoint_view()?; @@ -437,6 +508,9 @@ fn commit_fork_transaction( validator_set, generated_genesis, )?); + if let Some(test_account_rekey) = test_account_rekey { + writes.push(test_account_rekey_write(&view, test_account_rekey)?); + } let ledger_summary = db.reader.get_pre_committed_ledger_summary()?; let fork_txn = Transaction::GenesisTransaction(WriteSetPayload::Direct(ChangeSet::new( WriteSetMut::new(writes).freeze()?, @@ -460,6 +534,23 @@ fn commit_fork_transaction( Ok((fork_txn, waypoint)) } +fn test_account_rekey_write( + view: &impl TStateView, + rekey: &TestAccountRekey, +) -> Result<(StateKey, WriteOp)> { + let account_key = StateKey::resource_typed::(&rekey.address)?; + let account = view + .get_state_value_bytes(&account_key)? + .ok_or_else(|| format_err!("test account {} does not exist", rekey.address))?; + let mut account = bcs::from_bytes::(&account) + .with_context(|| format!("failed to decode Account resource for {}", rekey.address))?; + account.authentication_key = rekey.authentication_key.clone(); + Ok(( + account_key, + WriteOp::legacy_modification(bcs::to_bytes(&account)?.into()), + )) +} + fn validator_stake_writes( view: &impl TStateView, validator_set: &ValidatorSet, @@ -697,6 +788,22 @@ fn isolate_node_config(config: &mut NodeConfig) -> Result<()> { Ok(()) } +fn write_test_account_key(config_dir: &Path, rekey: &TestAccountRekey) -> Result<()> { + let private_key_path = config_dir.join(TEST_ACCOUNT_PRIVATE_KEY); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + options + .open(&private_key_path)? + .write_all(format!("0x{}\n", rekey.private_key_hex).as_bytes())?; + fs::write( + config_dir.join(TEST_ACCOUNT_ADDRESS), + format!("{}\n", rekey.address), + )?; + Ok(()) +} + fn write_manifest( config_dir: &Path, source_db_dir: &Path, @@ -707,7 +814,15 @@ fn write_manifest( fork_chain_id: ChainId, validator_addresses: &[AccountAddress], replacement_stake: u64, + test_account_rekey: Option<&TestAccountRekey>, ) -> Result { + let test_account_rekey = test_account_rekey.map(|rekey| { + json!({ + "address": rekey.address, + "authentication_key": format!("0x{}", hex::encode(&rekey.authentication_key)), + "private_key_path": config_dir.join(TEST_ACCOUNT_PRIVATE_KEY), + }) + }); let manifest = json!({ "source_db_dir": source_db_dir, "output_db_dir": output_db_dir, @@ -726,6 +841,7 @@ fn write_manifest( }, "generated_config_paths": node_config_paths, "waypoint": output_info.waypoint.map(|waypoint| waypoint.to_string()), + "test_account_rekey": test_account_rekey, }); let manifest_path = config_dir.join(MANIFEST); fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)?; @@ -952,6 +1068,8 @@ mod tests { fork_chain_id: fork_chain_id.id(), validators: 2, enable_storage_sharding: false, + test_account_address: None, + test_account_private_key: None, } .run() .unwrap(); @@ -1109,6 +1227,68 @@ mod tests { ); } + #[test] + fn fork_rekeys_existing_test_account_without_replacing_account_state() { + let (source_db_dir, _source_config_dir, root_key, _validators) = bootstrap_source_db(); + commit_non_reconfiguration_transaction(source_db_dir.path(), root_key); + let source_account = read_account_resource(source_db_dir.path(), AccountAddress::TWO) + .expect("test account must exist"); + let output_root = TempPath::new(); + output_root.create_as_dir().unwrap(); + let output_db_dir = output_root.path().join("fork-db"); + let config_dir = output_root.path().join("fork-configs"); + + Command { + source_db_dir: source_db_dir.path().to_path_buf(), + output_db_dir: output_db_dir.clone(), + config_dir: Some(config_dir.clone()), + fork_chain_id: 42, + validators: 1, + enable_storage_sharding: false, + test_account_address: Some(AccountAddress::TWO), + test_account_private_key: Some("0x1".to_string()), + } + .run() + .unwrap(); + + let output_account = read_account_resource(&output_db_dir, AccountAddress::TWO) + .expect("rekeyed account must exist"); + let expected = parse_test_account_rekey( + Some(AccountAddress::TWO), + Some("0x1".to_string()), + ) + .unwrap() + .unwrap(); + assert_eq!( + output_account.sequence_number(), + source_account.sequence_number() + ); + assert_eq!( + output_account.authentication_key(), + expected.authentication_key + ); + assert_ne!( + output_account.authentication_key(), + source_account.authentication_key() + ); + assert_eq!( + fs::read_to_string(config_dir.join(TEST_ACCOUNT_PRIVATE_KEY)).unwrap(), + format!("0x{}\n", expected.private_key_hex), + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(config_dir.join(TEST_ACCOUNT_PRIVATE_KEY)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600, + ); + } + } + #[tokio::test] #[ignore = "requires APTOS_NODE_BINARY pointing to an aptos-node binary"] async fn two_validator_fork_bootstraps_and_commits_blocks() { @@ -1141,6 +1321,8 @@ mod tests { fork_chain_id: 42, validators: 2, enable_storage_sharding: false, + test_account_address: None, + test_account_private_key: None, } .run() .unwrap(); From 88098bbc0f15c044f4974d6f82faf23476cec983 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 22 Jul 2026 22:10:28 -0700 Subject: [PATCH 08/21] Default fork test account key to 0xabc --- storage/db-tool/src/fork.rs | 355 ++++++++++++++++++++++-------------- 1 file changed, 222 insertions(+), 133 deletions(-) diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index d0dd7d3bd0f..f3558b8445a 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -23,15 +23,13 @@ use aptos_types::{ chain_id::ChainId, contract_event::ContractEvent, event::EventHandle, + network_address::{NetworkAddress, Protocol}, on_chain_config::{ CommitHistoryResource, ConfigurationResource, CurrentTimeMicroseconds, OnChainConfig, ValidatorSet, }, - network_address::{NetworkAddress, Protocol}, state_store::{state_key::StateKey, table::TableHandle, TStateView}, - transaction::{ - authenticator::AuthenticationKey, ChangeSet, Transaction, WriteSetPayload, - }, + transaction::{authenticator::AuthenticationKey, ChangeSet, Transaction, WriteSetPayload}, validator_config::ValidatorConfig, validator_performances::{ValidatorPerformance, ValidatorPerformances}, waypoint::Waypoint, @@ -45,6 +43,8 @@ use move_core_types::{ use rand::rngs::OsRng; use serde::{Deserialize, Serialize}; use serde_json::json; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; use std::{ collections::HashSet, fs::{self, File, OpenOptions}, @@ -54,14 +54,13 @@ use std::{ path::{Path, PathBuf}, sync::Arc, }; -#[cfg(unix)] -use std::os::unix::fs::OpenOptionsExt; const GENESIS_BLOB: &str = "genesis.blob"; const MANIFEST: &str = "fork-manifest.json"; const VALIDATOR_IDENTITY: &str = "validator-identity.yaml"; const TEST_ACCOUNT_ADDRESS: &str = "test-account-address"; const TEST_ACCOUNT_PRIVATE_KEY: &str = "test-account-private-key"; +const DEFAULT_TEST_ACCOUNT_PRIVATE_KEY: &str = "0xabc"; #[derive(Parser)] #[clap( @@ -87,129 +86,79 @@ pub struct Command { #[clap(long)] enable_storage_sharding: bool, - #[clap(long, requires = "test_account_private_key")] + /// Existing account to re-key for fork-only testing. + #[clap(long)] test_account_address: Option, + /// Private key for the fork test account. Defaults to 0xabc when an address is provided. #[clap(long, requires = "test_account_address")] test_account_private_key: Option, } impl Command { pub fn run(self) -> Result<()> { - let validators = NonZeroUsize::new(self.validators) - .ok_or_else(|| format_err!("--validators must be greater than 0"))?; - let fork_chain_id = validate_requested_chain_id(self.fork_chain_id)?; - let test_account_rekey = parse_test_account_rekey( - self.test_account_address, - self.test_account_private_key, - )?; - validate_output_paths(&self.source_db_dir, &self.output_db_dir)?; - let config_dir = self - .config_dir - .clone() - .unwrap_or_else(|| default_config_dir(&self.output_db_dir)); - validate_config_path(&self.source_db_dir, &self.output_db_dir, &config_dir)?; + let request = self.validate()?; let source_info = inspect_db(&self.source_db_dir, self.enable_storage_sharding, false) .with_context(|| format!("failed to inspect source DB {:?}", self.source_db_dir))?; ensure!( - fork_chain_id != source_info.chain_id, + request.chain_id != source_info.chain_id, "fork chain ID must differ from source chain ID {}", source_info.chain_id.id(), ); - fs::create_dir_all(&self.output_db_dir) - .with_context(|| format!("failed to create {:?}", self.output_db_dir))?; - AptosDB::create_checkpoint( + create_output_checkpoint( &self.source_db_dir, &self.output_db_dir, self.enable_storage_sharding, - ) - .with_context(|| { - format!( - "failed to create checkpoint from {:?} to {:?}", - self.source_db_dir, self.output_db_dir - ) - })?; - - fs::create_dir_all(&config_dir) - .with_context(|| format!("failed to create config dir {:?}", config_dir))?; - let replacement_stake = source_info - .validator_set - .active_validators - .iter() - .map(|validator| validator.consensus_voting_power()) - .max() - .ok_or_else(|| format_err!("source ValidatorSet has no active validators"))?; - let (_root_key, generated_genesis, _generated_waypoint, validators) = Builder::new( - &config_dir, - aptos_cached_packages::head_release_bundle().clone(), - )? - .with_num_validators(validators) - .with_init_genesis_stake(Some(Arc::new(move |_, stake| { - *stake = replacement_stake; - }))) - .build(OsRng)?; - let validator_set = validator_set_from_local_validators(&validators)?; - let generated_validator_addresses = validator_set.active_validators(); + )?; + let replacements = generate_replacement_validators( + &request.config_dir, + request.validator_count, + &source_info.validator_set, + )?; let (fork_txn, waypoint) = commit_fork_transaction( &self.output_db_dir, self.enable_storage_sharding, - fork_chain_id, - &validator_set, - &generated_genesis, - test_account_rekey.as_ref(), + request.chain_id, + &replacements.validator_set, + &replacements.genesis, + request.test_account_rekey.as_ref(), )?; let output_info = inspect_db(&self.output_db_dir, self.enable_storage_sharding, true) .with_context(|| format!("failed to inspect output DB {:?}", self.output_db_dir))?; - ensure!( - output_info.chain_id == fork_chain_id, - "output chain ID readback mismatch: expected {}, got {}", - fork_chain_id.id(), - output_info.chain_id.id(), - ); - ensure!( - output_info.validator_set == validator_set, - "output ValidatorSet readback mismatch", - ); - ensure!( - output_info.waypoint == Some(waypoint), - "output waypoint readback mismatch: manifest {}, DB {}", + verify_fork_output( + &source_info, + &output_info, + request.chain_id, + &replacements.validator_set, waypoint, - output_info - .waypoint - .map(|waypoint| waypoint.to_string()) - .unwrap_or_else(|| "".to_string()), - ); - ensure!( - output_info.validator_set != source_info.validator_set, - "fork did not replace ValidatorSet", - ); + )?; let node_config_paths = write_fork_configs( - &validators, + &replacements.configs, &self.output_db_dir, &fork_txn, waypoint, - fork_chain_id, + request.chain_id, self.enable_storage_sharding, )?; - if let Some(test_account_rekey) = &test_account_rekey { - write_test_account_key(&config_dir, test_account_rekey)?; + if let Some(test_account_rekey) = &request.test_account_rekey { + write_test_account_key(&request.config_dir, test_account_rekey)?; } let manifest_path = write_manifest( - &config_dir, + &request.config_dir, &self.source_db_dir, &self.output_db_dir, &node_config_paths, &source_info, &output_info, - fork_chain_id, - &generated_validator_addresses, - replacement_stake, - test_account_rekey.as_ref(), + request.chain_id, + &replacements.addresses, + replacements.stake, + request.test_account_rekey.as_ref(), )?; println!("Fork DB written to: {}", self.output_db_dir.display()); @@ -217,6 +166,43 @@ impl Command { println!("Fork manifest: {}", manifest_path.display()); Ok(()) } + + fn validate(&self) -> Result { + let validator_count = NonZeroUsize::new(self.validators) + .ok_or_else(|| format_err!("--validators must be greater than 0"))?; + let chain_id = validate_requested_chain_id(self.fork_chain_id)?; + let test_account_rekey = parse_test_account_rekey( + self.test_account_address, + self.test_account_private_key.clone(), + )?; + validate_output_paths(&self.source_db_dir, &self.output_db_dir)?; + let config_dir = self + .config_dir + .clone() + .unwrap_or_else(|| default_config_dir(&self.output_db_dir)); + validate_config_path(&self.source_db_dir, &self.output_db_dir, &config_dir)?; + Ok(ForkRequest { + chain_id, + validator_count, + config_dir, + test_account_rekey, + }) + } +} + +struct ForkRequest { + chain_id: ChainId, + validator_count: NonZeroUsize, + config_dir: PathBuf, + test_account_rekey: Option, +} + +struct ReplacementValidators { + configs: Vec, + genesis: Transaction, + validator_set: ValidatorSet, + addresses: Vec, + stake: u64, } #[derive(Clone)] @@ -274,16 +260,102 @@ struct TestAccountRekey { authentication_key: Vec, } +fn create_output_checkpoint( + source_db_dir: &Path, + output_db_dir: &Path, + enable_storage_sharding: bool, +) -> Result<()> { + fs::create_dir_all(output_db_dir) + .with_context(|| format!("failed to create {:?}", output_db_dir))?; + AptosDB::create_checkpoint(source_db_dir, output_db_dir, enable_storage_sharding).with_context( + || { + format!( + "failed to create checkpoint from {:?} to {:?}", + source_db_dir, output_db_dir + ) + }, + ) +} + +fn generate_replacement_validators( + config_dir: &Path, + validator_count: NonZeroUsize, + source_validator_set: &ValidatorSet, +) -> Result { + fs::create_dir_all(config_dir) + .with_context(|| format!("failed to create config dir {:?}", config_dir))?; + let stake = source_validator_set + .active_validators + .iter() + .map(|validator| validator.consensus_voting_power()) + .max() + .ok_or_else(|| format_err!("source ValidatorSet has no active validators"))?; + let (_root_key, genesis, _generated_waypoint, configs) = Builder::new( + config_dir, + aptos_cached_packages::head_release_bundle().clone(), + )? + .with_num_validators(validator_count) + .with_init_genesis_stake(Some(Arc::new(move |_, generated_stake| { + *generated_stake = stake; + }))) + .build(OsRng)?; + let validator_set = validator_set_from_local_validators(&configs)?; + let addresses = validator_set.active_validators(); + Ok(ReplacementValidators { + configs, + genesis, + validator_set, + addresses, + stake, + }) +} + +fn verify_fork_output( + source_info: &DbInfo, + output_info: &DbInfo, + chain_id: ChainId, + validator_set: &ValidatorSet, + waypoint: Waypoint, +) -> Result<()> { + ensure!( + output_info.chain_id == chain_id, + "output chain ID readback mismatch: expected {}, got {}", + chain_id.id(), + output_info.chain_id.id(), + ); + ensure!( + &output_info.validator_set == validator_set, + "output ValidatorSet readback mismatch", + ); + ensure!( + output_info.waypoint == Some(waypoint), + "output waypoint readback mismatch: manifest {}, DB {}", + waypoint, + output_info + .waypoint + .map(|waypoint| waypoint.to_string()) + .unwrap_or_else(|| "".to_string()), + ); + ensure!( + output_info.validator_set != source_info.validator_set, + "fork did not replace ValidatorSet", + ); + Ok(()) +} + fn parse_test_account_rekey( address: Option, private_key: Option, ) -> Result> { let (address, private_key) = match (address, private_key) { (None, None) => return Ok(None), - (Some(address), Some(private_key)) => (address, private_key), - _ => { + (Some(address), private_key) => ( + address, + private_key.unwrap_or_else(|| DEFAULT_TEST_ACCOUNT_PRIVATE_KEY.to_string()), + ), + (None, Some(_)) => { return Err(format_err!( - "--test-account-address and --test-account-private-key must be provided together" + "--test-account-private-key requires --test-account-address" )); }, }; @@ -399,7 +471,9 @@ fn inspect_db(db_dir: &Path, sharding: bool, require_epoch_waypoint: bool) -> Re let validator_set = ValidatorSet::fetch_config(&view).ok_or_else(|| format_err!("ValidatorSet missing"))?; let block_resource = view - .get_state_value_bytes(&StateKey::resource_typed::(&CORE_CODE_ADDRESS)?)? + .get_state_value_bytes(&StateKey::resource_typed::( + &CORE_CODE_ADDRESS, + )?)? .ok_or_else(|| format_err!("BlockResource missing"))?; let block_resource = bcs::from_bytes::(&block_resource)?; let current_time = CurrentTimeMicroseconds::fetch_config(&view) @@ -436,9 +510,7 @@ fn inspect_db(db_dir: &Path, sharding: bool, require_epoch_waypoint: bool) -> Re block_height: block_resource.height(), current_time_microseconds: current_time.microseconds, commit_history_length: commit_history.as_ref().map(CommitHistoryResource::length), - commit_history_next_idx: commit_history - .as_ref() - .map(CommitHistoryResource::next_idx), + commit_history_next_idx: commit_history.as_ref().map(CommitHistoryResource::next_idx), }) } @@ -721,12 +793,8 @@ fn write_fork_configs( validator.index, validator_db_dir ) })?; - AptosDB::create_checkpoint( - output_db_dir, - &validator_db_dir, - enable_storage_sharding, - ) - .with_context(|| { + AptosDB::create_checkpoint(output_db_dir, &validator_db_dir, enable_storage_sharding) + .with_context(|| { format!( "failed to create validator {} DB checkpoint at {:?}", validator.index, validator_db_dir @@ -775,10 +843,11 @@ fn isolate_node_config(config: &mut NodeConfig) -> Result<()> { .as_slice() .iter() .map(|protocol| match protocol { - Protocol::Ip4(_) | Protocol::Ip6(_) | Protocol::Dns(_) - | Protocol::Dns4(_) | Protocol::Dns6(_) => { - Protocol::Ip4(Ipv4Addr::LOCALHOST) - }, + Protocol::Ip4(_) + | Protocol::Ip6(_) + | Protocol::Dns(_) + | Protocol::Dns4(_) + | Protocol::Dns6(_) => Protocol::Ip4(Ipv4Addr::LOCALHOST), protocol => protocol.clone(), }) .collect(); @@ -875,9 +944,9 @@ mod tests { use aptos_executor_types::BlockExecutorTrait; use aptos_temppath::TempPath; use aptos_types::{ + account_config::AccountResource, aggregate_signature::AggregateSignature, block_info::BlockInfo, - account_config::AccountResource, ledger_info::{LedgerInfo, LedgerInfoWithSignatures}, test_helpers::transaction_test_helpers::{ block, get_test_signed_transaction, TEST_BLOCK_EXECUTOR_ONCHAIN_CONFIG, @@ -1049,11 +1118,9 @@ mod tests { .first() .unwrap(); let source_validator_address = source_validator.account_address; - let source_validator_config = read_validator_config( - source_db_dir.path(), - source_validator_address, - ) - .expect("source validator config must exist before fork"); + let source_validator_config = + read_validator_config(source_db_dir.path(), source_validator_address) + .expect("source validator config must exist before fork"); assert_eq!(&source_validator_config, source_validator.config()); let output_root = TempPath::new(); output_root.create_as_dir().unwrap(); @@ -1139,9 +1206,7 @@ mod tests { source_account, ); for validator_index in 0..2 { - let validator_db_dir = config_dir - .join(validator_index.to_string()) - .join("fork-db"); + let validator_db_dir = config_dir.join(validator_index.to_string()).join("fork-db"); let validator_info = inspect_db(&validator_db_dir, false, true).unwrap(); assert_eq!(validator_info.chain_id, fork_chain_id); assert_eq!(validator_info.validator_set, output_info.validator_set); @@ -1173,11 +1238,7 @@ mod tests { ))); } - let output_validator = output_info - .validator_set - .active_validators - .first() - .unwrap(); + let output_validator = output_info.validator_set.active_validators.first().unwrap(); let output_validator_address = output_validator.account_address; assert_ne!(output_validator_address, source_validator_address); assert!( @@ -1205,7 +1266,8 @@ mod tests { let bytes = fs::read(config_dir.join("0").join(GENESIS_BLOB)).unwrap(); bcs::from_bytes::(&bytes).unwrap() }; - let Transaction::GenesisTransaction(WriteSetPayload::Direct(change_set)) = genesis_txn else { + let Transaction::GenesisTransaction(WriteSetPayload::Direct(change_set)) = genesis_txn + else { panic!("fork transaction must be a direct genesis write-set"); }; let new_block_event = change_set @@ -1218,7 +1280,10 @@ mod tests { source_info_before.new_block_event_count ); let new_block_event = NewBlockEvent::try_from_bytes(new_block_event.event_data()).unwrap(); - assert_eq!(new_block_event.height(), source_info_before.new_block_event_count); + assert_eq!( + new_block_event.height(), + source_info_before.new_block_event_count + ); assert_eq!(new_block_event.round(), u64::MAX); assert_eq!(new_block_event.proposer(), AccountAddress::ZERO); assert_eq!( @@ -1246,19 +1311,17 @@ mod tests { validators: 1, enable_storage_sharding: false, test_account_address: Some(AccountAddress::TWO), - test_account_private_key: Some("0x1".to_string()), + test_account_private_key: None, } .run() .unwrap(); let output_account = read_account_resource(&output_db_dir, AccountAddress::TWO) .expect("rekeyed account must exist"); - let expected = parse_test_account_rekey( - Some(AccountAddress::TWO), - Some("0x1".to_string()), - ) - .unwrap() - .unwrap(); + let expected = parse_test_account_rekey(Some(AccountAddress::TWO), None) + .unwrap() + .unwrap(); + assert!(expected.private_key_hex.ends_with("0abc")); assert_eq!( output_account.sequence_number(), source_account.sequence_number() @@ -1289,6 +1352,33 @@ mod tests { } } + #[test] + fn test_account_private_key_defaults_to_abc_and_allows_override() { + let default = parse_test_account_rekey(Some(AccountAddress::TWO), None) + .unwrap() + .unwrap(); + let explicit_default = parse_test_account_rekey( + Some(AccountAddress::TWO), + Some(DEFAULT_TEST_ACCOUNT_PRIVATE_KEY.to_string()), + ) + .unwrap() + .unwrap(); + let override_key = + parse_test_account_rekey(Some(AccountAddress::TWO), Some("0xdef".to_string())) + .unwrap() + .unwrap(); + + assert_eq!(default.private_key_hex, explicit_default.private_key_hex); + assert_eq!( + default.authentication_key, + explicit_default.authentication_key + ); + assert!(default.private_key_hex.ends_with("0abc")); + assert!(override_key.private_key_hex.ends_with("0def")); + assert_ne!(default.authentication_key, override_key.authentication_key); + assert!(parse_test_account_rekey(None, Some("0xabc".to_string())).is_err()); + } + #[tokio::test] #[ignore = "requires APTOS_NODE_BINARY pointing to an aptos-node binary"] async fn two_validator_fork_bootstraps_and_commits_blocks() { @@ -1352,15 +1442,14 @@ mod tests { let deadline = tokio::time::Instant::now() + Duration::from_secs(60); while tokio::time::Instant::now() < deadline { for child in &mut nodes.0 { - assert!(child.try_wait().unwrap().is_none(), "validator exited early"); + assert!( + child.try_wait().unwrap().is_none(), + "validator exited early" + ); } let mut ledger_infos = Vec::new(); for address in &api_addresses { - if let Ok(response) = client - .get(format!("http://{}/v1/", address)) - .send() - .await - { + if let Ok(response) = client.get(format!("http://{}/v1/", address)).send().await { if let Ok(info) = response.json::().await { ledger_infos.push(info); } From a909f218ba9767b951e72b5850ccbd15173c7818 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 27 Jul 2026 14:59:53 -0700 Subject: [PATCH 09/21] Add ephemeral fork network launcher --- scripts/ephemeral_fork.sh | 506 +++++++++++++++++++++++++++ scripts/tests/ephemeral_fork_test.sh | 203 +++++++++++ 2 files changed, 709 insertions(+) create mode 100755 scripts/ephemeral_fork.sh create mode 100755 scripts/tests/ephemeral_fork_test.sh diff --git a/scripts/ephemeral_fork.sh b/scripts/ephemeral_fork.sh new file mode 100755 index 00000000000..d36f6172037 --- /dev/null +++ b/scripts/ephemeral_fork.sh @@ -0,0 +1,506 @@ +#!/usr/bin/env bash + +# Copyright © Aptos Foundation +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +PROGRAM=$(basename "$0") +DEFAULT_CHAIN_ID=42 +DEFAULT_VALIDATORS=2 +DEFAULT_HEALTH_TIMEOUT=120 +STATE_DIR_NAME=".ephemeral-fork" + +usage() { + cat <&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} + +resolve_binary() { + local binary=$1 + local resolved + if [[ "$binary" == */* ]]; then + [[ -x "$binary" ]] || die "binary is not executable: $binary" + resolved=$(cd "$(dirname "$binary")" && pwd -P)/$(basename "$binary") + else + resolved=$(command -v "$binary") || die "binary not found: $binary" + fi + printf '%s\n' "$resolved" +} + +validate_positive_integer() { + local name=$1 + local value=$2 + [[ "$value" =~ ^[1-9][0-9]*$ ]] || die "$name must be a positive integer" +} + +validate_chain_id() { + local chain_id=$1 + validate_positive_integer "--chain-id" "$chain_id" + ((chain_id > 1 && chain_id <= 255)) || + die "--chain-id must be between 2 and 255; 1 is reserved for mainnet" +} + +validate_account_address() { + local address=$1 + [[ "$address" =~ ^0x[0-9a-fA-F]{1,64}$ ]] || + die "--test-account-address must be a 0x-prefixed account address" +} + +state_path() { + local work_dir=$1 + local name=$2 + printf '%s/%s/%s\n' "$work_dir" "$STATE_DIR_NAME" "$name" +} + +read_state() { + local work_dir=$1 + local name=$2 + local path + path=$(state_path "$work_dir" "$name") + [[ -f "$path" ]] || die "missing fork state file: $path" + cat "$path" +} + +api_address_from_config() { + local config=$1 + awk ' + /^api:$/ { in_api = 1; next } + in_api && /^[^ ]/ { in_api = 0 } + in_api && $1 == "address:" { + value = $2 + gsub(/^"/, "", value) + gsub(/"$/, "", value) + print value + exit + } + ' "$config" +} + +ensure_loopback_address() { + local address=$1 + case "$address" in + 127.0.0.1:* | localhost:* | "[::1]:"*) ;; + *) die "generated REST address is not loopback-only: $address" ;; + esac +} + +ledger_summary() { + python3 -c ' +import json +import sys + +data = json.load(sys.stdin) +print( + "chain_id={chain_id} epoch={epoch} ledger_version={ledger_version}".format( + chain_id=data["chain_id"], + epoch=data["epoch"], + ledger_version=data["ledger_version"], + ) +) +' +} + +ledger_version() { + python3 -c 'import json, sys; print(json.load(sys.stdin)["ledger_version"])' +} + +manifest_output_version() { + local manifest=$1 + python3 -c ' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as manifest: + print(json.load(manifest)["output_ledger"]["version"]) +' "$manifest" +} + +stop_nodes() { + local config_dir=$1 + local quiet=${2:-false} + local config pid_file pid command attempt + + for config in "$config_dir"/*/node.yaml; do + [[ -f "$config" ]] || continue + pid_file="$(dirname "$config")/node.pid" + [[ -f "$pid_file" ]] || continue + pid=$(cat "$pid_file") + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || die "invalid PID in $pid_file" + + if ! kill -0 "$pid" 2>/dev/null; then + [[ "$quiet" == true ]] || echo "Validator PID $pid is already stopped." + continue + fi + + command=$(ps -p "$pid" -o command=) + case "$command" in + *"$config"*) ;; + *) die "refusing to stop PID $pid because it is not using $config" ;; + esac + + kill "$pid" + for ((attempt = 0; attempt < 50; attempt++)); do + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + done + if kill -0 "$pid" 2>/dev/null; then + die "validator PID $pid did not stop after SIGTERM" + fi + [[ "$quiet" == true ]] || echo "Stopped validator PID $pid." + done +} + +wait_for_network() { + local config_dir=$1 + local rest_urls_file=$2 + local fork_version=$3 + local timeout=$4 + local deadline=$((SECONDS + timeout)) + local index pid response version all_ready + + while ((SECONDS < deadline)); do + all_ready=true + index=0 + while IFS= read -r url; do + pid=$(cat "$config_dir/$index/node.pid") + if ! kill -0 "$pid" 2>/dev/null; then + echo "Validator $index exited during startup:" >&2 + tail -n 80 "$config_dir/$index/node.log" >&2 || true + return 1 + fi + response=$(curl --fail --silent --max-time 2 "$url/v1/" 2>/dev/null || true) + if [[ -z "$response" ]]; then + all_ready=false + else + version=$(printf '%s' "$response" | ledger_version) + if ((version <= fork_version)); then + all_ready=false + fi + fi + index=$((index + 1)) + done <"$rest_urls_file" + + if [[ "$all_ready" == true ]]; then + return 0 + fi + sleep 2 + done + + echo "Timed out waiting for all validators to advance beyond version $fork_version." >&2 + return 1 +} + +print_handoff() { + local work_dir=$1 + local config_dir=$2 + local rest_url=$3 + local test_account=$4 + local key_file="$config_dir/test-account-private-key" + local manifest="$config_dir/fork-manifest.json" + + echo + echo "Ephemeral fork is healthy." + echo " Work directory: $work_dir" + echo " REST URL: $rest_url" + echo " Test account: $test_account" + echo " Private key file: $key_file" + echo " Fork manifest: $manifest" + echo + echo "The private key is public test material and must never be used on production." + echo + echo "Contract publishing pattern:" + echo " movement move publish --package-dir \\" + echo " --sender-account $test_account \\" + echo " --private-key-file $key_file --url $rest_url --assume-yes" + echo + echo "Status: $PROGRAM status --work-dir $work_dir" + echo "Stop: $PROGRAM stop --work-dir $work_dir" +} + +start_fork() { + local source_db="" + local source_is_disposable=false + local fork_tool="" + local node_binary="" + local test_account="" + local test_private_key="" + local work_dir="" + local validators=$DEFAULT_VALIDATORS + local chain_id=$DEFAULT_CHAIN_ID + local health_timeout=$DEFAULT_HEALTH_TIMEOUT + + while (($#)); do + case "$1" in + --source-db) + [[ $# -ge 2 ]] || die "--source-db requires a value" + source_db=$2 + shift 2 + ;; + --source-is-disposable) + source_is_disposable=true + shift + ;; + --fork-tool) + [[ $# -ge 2 ]] || die "--fork-tool requires a value" + fork_tool=$2 + shift 2 + ;; + --node-binary) + [[ $# -ge 2 ]] || die "--node-binary requires a value" + node_binary=$2 + shift 2 + ;; + --test-account-address) + [[ $# -ge 2 ]] || die "--test-account-address requires a value" + test_account=$2 + shift 2 + ;; + --test-account-private-key) + [[ $# -ge 2 ]] || die "--test-account-private-key requires a value" + test_private_key=$2 + shift 2 + ;; + --work-dir) + [[ $# -ge 2 ]] || die "--work-dir requires a value" + work_dir=$2 + shift 2 + ;; + --validators) + [[ $# -ge 2 ]] || die "--validators requires a value" + validators=$2 + shift 2 + ;; + --chain-id) + [[ $# -ge 2 ]] || die "--chain-id requires a value" + chain_id=$2 + shift 2 + ;; + --health-timeout) + [[ $# -ge 2 ]] || die "--health-timeout requires a value" + health_timeout=$2 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown start option: $1" + ;; + esac + done + + [[ -n "$source_db" ]] || die "--source-db is required" + [[ "$source_is_disposable" == true ]] || + die "--source-is-disposable is required; the source DB may be recovered or mutated" + [[ -d "$source_db" ]] || die "source DB directory does not exist: $source_db" + [[ -w "$source_db" ]] || die "source DB must be writable and disposable: $source_db" + [[ -n "$fork_tool" ]] || die "--fork-tool is required" + [[ -n "$node_binary" ]] || die "--node-binary is required" + [[ -n "$test_account" ]] || die "--test-account-address is required for contract testing" + + require_command curl + require_command nohup + require_command ps + require_command python3 + validate_positive_integer "--validators" "$validators" + validate_positive_integer "--health-timeout" "$health_timeout" + validate_chain_id "$chain_id" + validate_account_address "$test_account" + fork_tool=$(resolve_binary "$fork_tool") + node_binary=$(resolve_binary "$node_binary") + source_db=$(cd "$source_db" && pwd -P) + + if [[ -z "$work_dir" ]]; then + work_dir=$(mktemp -d "${TMPDIR:-/tmp}/aptos-ephemeral-fork.XXXXXX") + else + mkdir -p "$work_dir" + [[ -z "$(ls -A "$work_dir")" ]] || die "work directory must be empty: $work_dir" + work_dir=$(cd "$work_dir" && pwd -P) + fi + + local output_db="$work_dir/fork-db" + local config_dir="$work_dir/configs" + local state_dir="$work_dir/$STATE_DIR_NAME" + local manifest="$config_dir/fork-manifest.json" + local rest_urls_file="$state_dir/rest-urls" + local fork_command=( + "$fork_tool" aptos-db fork + --source-db-dir "$source_db" + --output-db-dir "$output_db" + --config-dir "$config_dir" + --fork-chain-id "$chain_id" + --validators "$validators" + --test-account-address "$test_account" + ) + if [[ -n "$test_private_key" ]]; then + fork_command+=(--test-account-private-key "$test_private_key") + fi + + echo "Creating fork in $work_dir..." + "${fork_command[@]}" + [[ -f "$manifest" ]] || die "fork tool did not create manifest: $manifest" + [[ -f "$config_dir/test-account-private-key" ]] || + die "fork tool did not create the test account key" + + mkdir -p "$state_dir" + printf '%s\n' "$config_dir" >"$state_dir/config-dir" + printf '%s\n' "$node_binary" >"$state_dir/node-binary" + printf '%s\n' "$chain_id" >"$state_dir/chain-id" + printf '%s\n' "$test_account" >"$state_dir/test-account" + : >"$rest_urls_file" + + local start_complete=false + trap 'if [[ "$start_complete" != true ]]; then stop_nodes "$config_dir" true || true; fi' EXIT + trap 'exit 130' INT TERM + + local index config api_address rest_url + for ((index = 0; index < validators; index++)); do + config="$config_dir/$index/node.yaml" + [[ -f "$config" ]] || die "missing generated node config: $config" + api_address=$(api_address_from_config "$config") + [[ -n "$api_address" ]] || die "could not read REST address from $config" + ensure_loopback_address "$api_address" + rest_url="http://$api_address" + printf '%s\n' "$rest_url" >>"$rest_urls_file" + + nohup "$node_binary" -f "$config" >"$config_dir/$index/node.log" 2>&1 "$config_dir/$index/node.pid" + done + + local fork_version + fork_version=$(manifest_output_version "$manifest") + echo "Waiting for $validators validators to advance beyond fork version $fork_version..." + wait_for_network "$config_dir" "$rest_urls_file" "$fork_version" "$health_timeout" + + start_complete=true + trap - EXIT INT TERM + print_handoff "$work_dir" "$config_dir" "$(head -n 1 "$rest_urls_file")" "$test_account" +} + +status_fork() { + local work_dir="" + while (($#)); do + case "$1" in + --work-dir) + [[ $# -ge 2 ]] || die "--work-dir requires a value" + work_dir=$2 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown status option: $1" + ;; + esac + done + [[ -n "$work_dir" ]] || die "--work-dir is required" + [[ -d "$work_dir" ]] || die "work directory does not exist: $work_dir" + + require_command curl + require_command python3 + local config_dir rest_urls_file index=0 url pid response failed=false + config_dir=$(read_state "$work_dir" config-dir) + rest_urls_file=$(state_path "$work_dir" rest-urls) + [[ -f "$rest_urls_file" ]] || die "missing REST URL state: $rest_urls_file" + + while IFS= read -r url; do + pid=$(cat "$config_dir/$index/node.pid") + if ! kill -0 "$pid" 2>/dev/null; then + echo "validator=$index pid=$pid status=stopped" + failed=true + else + response=$(curl --fail --silent --max-time 2 "$url/v1/" 2>/dev/null || true) + if [[ -z "$response" ]]; then + echo "validator=$index pid=$pid status=unhealthy rest_url=$url" + failed=true + else + printf 'validator=%s pid=%s status=healthy rest_url=%s ' "$index" "$pid" "$url" + printf '%s' "$response" | ledger_summary + fi + fi + index=$((index + 1)) + done <"$rest_urls_file" + + [[ "$failed" == false ]] +} + +stop_fork() { + local work_dir="" + while (($#)); do + case "$1" in + --work-dir) + [[ $# -ge 2 ]] || die "--work-dir requires a value" + work_dir=$2 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown stop option: $1" + ;; + esac + done + [[ -n "$work_dir" ]] || die "--work-dir is required" + [[ -d "$work_dir" ]] || die "work directory does not exist: $work_dir" + + local config_dir + config_dir=$(read_state "$work_dir" config-dir) + stop_nodes "$config_dir" +} + +main() { + local action=${1:-} + if [[ -z "$action" ]]; then + usage + exit 1 + fi + shift + + case "$action" in + start) start_fork "$@" ;; + status) status_fork "$@" ;; + stop) stop_fork "$@" ;; + -h | --help | help) usage ;; + *) die "unknown command: $action" ;; + esac +} + +main "$@" diff --git a/scripts/tests/ephemeral_fork_test.sh b/scripts/tests/ephemeral_fork_test.sh new file mode 100755 index 00000000000..945a29a9582 --- /dev/null +++ b/scripts/tests/ephemeral_fork_test.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +# Copyright © Aptos Foundation +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/../.." && pwd -P) +LAUNCHER="$ROOT/scripts/ephemeral_fork.sh" +TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/ephemeral-fork-test.XXXXXX") +SOURCE_DB="$TEST_ROOT/source-db" +WORK_DIR="$TEST_ROOT/work" +MOCK_FORK_TOOL="$TEST_ROOT/mock-aptos-debugger" +MOCK_NODE="$TEST_ROOT/mock-aptos-node" + +cleanup() { + if [[ -d "$WORK_DIR/.ephemeral-fork" ]]; then + "$LAUNCHER" stop --work-dir "$WORK_DIR" >/dev/null 2>&1 || true + fi + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +mkdir -p "$SOURCE_DB" + +read -r MOCK_PORT_0 MOCK_PORT_1 < <( + python3 -c ' +import socket + +sockets = [] +ports = [] +for _ in range(2): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sockets.append(sock) + ports.append(sock.getsockname()[1]) +print(*ports) +' +) +export MOCK_PORT_0 MOCK_PORT_1 + +cat >"$MOCK_FORK_TOOL" <<'MOCK_FORK' +#!/usr/bin/env bash +set -euo pipefail + +[[ "$1 $2 $3" == "aptos-db fork --source-db-dir" ]] +shift 2 +output_db="" +config_dir="" +validators="" +test_account="" +while (($#)); do + case "$1" in + --source-db-dir | --fork-chain-id) + shift 2 + ;; + --output-db-dir) + output_db=$2 + shift 2 + ;; + --config-dir) + config_dir=$2 + shift 2 + ;; + --validators) + validators=$2 + shift 2 + ;; + --test-account-address) + test_account=$2 + shift 2 + ;; + --test-account-private-key) + shift 2 + ;; + *) + echo "unexpected mock fork argument: $1" >&2 + exit 1 + ;; + esac +done + +mkdir -p "$output_db" "$config_dir" +for ((index = 0; index < validators; index++)); do + port_name="MOCK_PORT_$index" + port=${!port_name} + mkdir -p "$config_dir/$index" + cat >"$config_dir/$index/node.yaml" <"$config_dir/test-account-address" +printf '%s\n' \ + "0x0000000000000000000000000000000000000000000000000000000000000abc" \ + >"$config_dir/test-account-private-key" +chmod 600 "$config_dir/test-account-private-key" +cat >"$config_dir/fork-manifest.json" <"$MOCK_NODE" <<'MOCK_NODE' +#!/usr/bin/env python3 +import argparse +import http.server +import json +import re + +parser = argparse.ArgumentParser() +parser.add_argument("-f", dest="config", required=True) +args = parser.parse_args() +with open(args.config, encoding="utf-8") as config_file: + config = config_file.read() +match = re.search(r'address: "127\.0\.0\.1:(\d+)"', config) +if match is None: + raise RuntimeError("mock node config has no REST port") +port = int(match.group(1)) + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = json.dumps( + { + "chain_id": 42, + "epoch": "7", + "ledger_version": "11", + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + return + + +http.server.HTTPServer(("127.0.0.1", port), Handler).serve_forever() +MOCK_NODE +chmod +x "$MOCK_NODE" + +start_output=$( + "$LAUNCHER" start \ + --source-db "$SOURCE_DB" \ + --source-is-disposable \ + --fork-tool "$MOCK_FORK_TOOL" \ + --node-binary "$MOCK_NODE" \ + --test-account-address 0x1234 \ + --work-dir "$WORK_DIR" \ + --health-timeout 10 +) +grep -q "Ephemeral fork is healthy." <<<"$start_output" +grep -q "Private key file:" <<<"$start_output" +grep -q "movement move publish" <<<"$start_output" + +status_output=$("$LAUNCHER" status --work-dir "$WORK_DIR") +[[ $(grep -c "status=healthy" <<<"$status_output") -eq 2 ]] +grep -q "chain_id=42" <<<"$status_output" +grep -q "ledger_version=11" <<<"$status_output" + +expected_key="0x0000000000000000000000000000000000000000000000000000000000000abc" +[[ $(cat "$WORK_DIR/configs/test-account-private-key") == "$expected_key" ]] + +"$LAUNCHER" stop --work-dir "$WORK_DIR" >/dev/null +if "$LAUNCHER" status --work-dir "$WORK_DIR" >/dev/null 2>&1; then + echo "status unexpectedly succeeded after validators stopped" >&2 + exit 1 +fi + +if "$LAUNCHER" start \ + --source-db "$SOURCE_DB" \ + --fork-tool "$MOCK_FORK_TOOL" \ + --node-binary "$MOCK_NODE" \ + --test-account-address 0x1234 \ + --work-dir "$TEST_ROOT/missing-ack" >/dev/null 2>&1; then + echo "start unexpectedly accepted a source without disposable acknowledgement" >&2 + exit 1 +fi + +if "$LAUNCHER" start \ + --source-db "$SOURCE_DB" \ + --source-is-disposable \ + --fork-tool "$MOCK_FORK_TOOL" \ + --node-binary "$MOCK_NODE" \ + --test-account-address 0x1234 \ + --chain-id 1 \ + --work-dir "$TEST_ROOT/mainnet-chain-id" >/dev/null 2>&1; then + echo "start unexpectedly accepted mainnet chain ID 1" >&2 + exit 1 +fi + +echo "ephemeral fork lifecycle test passed" From 6773017a3441f61ef68859fbec9142a2162ff318 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 27 Jul 2026 15:21:11 -0700 Subject: [PATCH 10/21] Add SHA-based remote mainnet fork launcher --- scripts/remote/start_ephemeral_mainnet.sh | 267 ++++++++++++++++++ scripts/start_remote_ephemeral_mainnet.sh | 216 ++++++++++++++ .../tests/remote_ephemeral_mainnet_test.sh | 67 +++++ 3 files changed, 550 insertions(+) create mode 100755 scripts/remote/start_ephemeral_mainnet.sh create mode 100755 scripts/start_remote_ephemeral_mainnet.sh create mode 100755 scripts/tests/remote_ephemeral_mainnet_test.sh diff --git a/scripts/remote/start_ephemeral_mainnet.sh b/scripts/remote/start_ephemeral_mainnet.sh new file mode 100755 index 00000000000..3c37252675a --- /dev/null +++ b/scripts/remote/start_ephemeral_mainnet.sh @@ -0,0 +1,267 @@ +#!/usr/bin/env bash + +# Copyright © Aptos Foundation +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +PROGRAM=$(basename "$0") +RUN_ROOT=${EPHEMERAL_RUN_ROOT:-/mnt/mainnet-volume/fork-run-20260722} +SERVICE_ROOT=${EPHEMERAL_SERVICE_ROOT:-/mnt/mainnet-volume/ephemeral-mainnet} +SOURCE_DB=${EPHEMERAL_SOURCE_DB:-$RUN_ROOT/staging-db-v5} +FORK_TOOL=${EPHEMERAL_FORK_TOOL:-$RUN_ROOT/target/release/aptos-debugger} +FORK_LAUNCHER=${EPHEMERAL_FORK_LAUNCHER:-$RUN_ROOT/bin/ephemeral_fork.sh} +CARGO_HOME=${CARGO_HOME:-$RUN_ROOT/cargo} +RUSTUP_HOME=${RUSTUP_HOME:-$RUN_ROOT/rustup} +TEST_ACCOUNT=${EPHEMERAL_TEST_ACCOUNT:-0x573537299646e0dfab6ca81086edccf73f77b841f30dde6bfbd730ed428479bf} +VALIDATORS=${EPHEMERAL_VALIDATORS:-2} +CHAIN_ID=${EPHEMERAL_CHAIN_ID:-42} +HEALTH_TIMEOUT=${EPHEMERAL_HEALTH_TIMEOUT:-180} +ALREADY_RUNNING_EXIT=3 +export PATH="$CARGO_HOME/bin:$PATH" + +usage() { + cat <&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} + +api_address_from_config() { + local config=$1 + awk ' + /^api:$/ { in_api = 1; next } + in_api && /^[^ ]/ { in_api = 0 } + in_api && $1 == "address:" { + value = $2 + gsub(/^"/, "", value) + gsub(/"$/, "", value) + print value + exit + } + ' "$config" +} + +config_from_command() { + local command=$1 + sed -n 's/.*[[:space:]]-f[[:space:]]\([^[:space:]]*\).*/\1/p' <<<"$command" +} + +report_running_network() { + local pids + pids=$(pgrep -x aptos-node || true) + [[ -n "$pids" ]] || return 1 + + local active_sha="unknown" + if [[ -f "$SERVICE_ROOT/active/sha" ]]; then + active_sha=$(cat "$SERVICE_ROOT/active/sha") + fi + echo "An ephemeral network is already running; refusing to replace it." + echo " Recorded SHA: $active_sha" + + local pid command config api response + while IFS= read -r pid; do + command=$(ps -p "$pid" -o command=) + config=$(config_from_command "$command") + echo " PID $pid: $command" + if [[ -n "$config" && -f "$config" ]]; then + api=$(api_address_from_config "$config") + if [[ -n "$api" ]]; then + response=$(curl --fail --silent --max-time 2 "http://$api/v1/" 2>/dev/null || true) + if [[ -n "$response" ]]; then + python3 -c ' +import json +import sys + +info = json.load(sys.stdin) +print( + " REST=http://{api} chain_id={chain_id} epoch={epoch} ledger_version={version}".format( + api=sys.argv[1], + chain_id=info["chain_id"], + epoch=info["epoch"], + version=info["ledger_version"], + ) +) +' "$api" <<<"$response" + else + echo " REST=http://$api status=unhealthy" + fi + fi + fi + done <<<"$pids" + return 0 +} + +active_api_endpoint() { + local pids pid command config api + pids=$(pgrep -x aptos-node || true) + [[ -n "$pids" ]] || die "no aptos-node process is running" + while IFS= read -r pid; do + command=$(ps -p "$pid" -o command=) + config=$(config_from_command "$command") + if [[ -n "$config" && -f "$config" ]]; then + api=$(api_address_from_config "$config") + if [[ "$api" == 127.0.0.1:* ]]; then + printf '%s\n' "$api" + return + fi + fi + done <<<"$pids" + die "running validators do not expose a loopback REST endpoint" +} + +active_test_key() { + local work_dir + [[ -f "$SERVICE_ROOT/active/work-dir" ]] || + die "active network metadata does not contain a work directory" + work_dir=$(cat "$SERVICE_ROOT/active/work-dir") + local key_file="$work_dir/configs/test-account-private-key" + [[ -f "$key_file" ]] || die "active network test key is missing: $key_file" + cat "$key_file" +} + +preflight() { + mkdir -p "$SERVICE_ROOT" + exec 9>"$SERVICE_ROOT/launcher.lock" + flock 9 + if report_running_network; then + return "$ALREADY_RUNNING_EXIT" + fi + echo "No aptos-node process is running; remote host is ready." +} + +verify_archive() { + local expected_sha=$1 + local archive=$2 + [[ -f "$archive" ]] || die "source archive does not exist: $archive" + + local archive_sha + archive_sha=$( + gzip -cd "$archive" | + { + git get-tar-commit-id + cat >/dev/null + } + ) || + die "source archive does not contain a Git commit ID" + [[ "$archive_sha" == "$expected_sha" ]] || + die "archive commit $archive_sha does not match requested SHA $expected_sha" +} + +build_node() { + local sha=$1 + local archive=$2 + local build_root="$SERVICE_ROOT/builds/$sha" + local source_dir="$build_root/source" + local target_dir="$build_root/target" + local node_binary="$target_dir/release/aptos-node" + + if [[ -x "$node_binary" ]]; then + echo "Reusing aptos-node already built for $sha." >&2 + printf '%s\n' "$node_binary" + return + fi + + if [[ -e "$source_dir" ]]; then + mv "$source_dir" "$source_dir.incomplete.$(date +%s)" + fi + mkdir -p "$source_dir" "$target_dir" + tar -xzf "$archive" -C "$source_dir" + + echo "Building aptos-node for m1 commit $sha..." >&2 + ( + export CARGO_HOME RUSTUP_HOME + export CARGO_TARGET_DIR="$target_dir" + export CXXFLAGS="${CXXFLAGS:--include cstdint}" + cd "$source_dir" + cargo build --release -p aptos-node + ) + [[ -x "$node_binary" ]] || die "build did not produce $node_binary" + printf '%s\n' "$node_binary" +} + +start_network() { + local sha=${1:-} + local archive=${2:-} + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || die "start requires a full lowercase GitHub SHA" + [[ -n "$archive" ]] || die "start requires a source archive path" + + require_command cargo + require_command curl + require_command flock + require_command git + require_command gzip + require_command pgrep + require_command ps + require_command python3 + require_command tar + [[ -d "$SOURCE_DB" ]] || die "disposable source DB is missing: $SOURCE_DB" + [[ -w "$SOURCE_DB" ]] || die "disposable source DB is not writable: $SOURCE_DB" + [[ -x "$FORK_TOOL" ]] || die "fork tool is missing or not executable: $FORK_TOOL" + [[ -x "$FORK_LAUNCHER" ]] || die "fork launcher is missing or not executable: $FORK_LAUNCHER" + + mkdir -p "$SERVICE_ROOT" + exec 9>"$SERVICE_ROOT/launcher.lock" + flock 9 + if report_running_network; then + return "$ALREADY_RUNNING_EXIT" + fi + + verify_archive "$sha" "$archive" + local node_binary + node_binary=$(build_node "$sha" "$archive") + local work_dir + work_dir="$SERVICE_ROOT/runs/$sha-$(date -u +%Y%m%dT%H%M%SZ)" + mkdir -p "$(dirname "$work_dir")" + + "$FORK_LAUNCHER" start \ + --source-db "$SOURCE_DB" \ + --source-is-disposable \ + --fork-tool "$FORK_TOOL" \ + --node-binary "$node_binary" \ + --test-account-address "$TEST_ACCOUNT" \ + --validators "$VALIDATORS" \ + --chain-id "$CHAIN_ID" \ + --health-timeout "$HEALTH_TIMEOUT" \ + --work-dir "$work_dir" + + mkdir -p "$SERVICE_ROOT/active" + printf '%s\n' "$sha" >"$SERVICE_ROOT/active/sha" + printf '%s\n' "$work_dir" >"$SERVICE_ROOT/active/work-dir" + echo "Remote ephemeral mainnet started from m1 commit $sha." +} + +main() { + local action=${1:-} + shift || true + case "$action" in + preflight) preflight "$@" ;; + start) start_network "$@" ;; + endpoint) active_api_endpoint "$@" ;; + test-key) active_test_key "$@" ;; + -h | --help | help) usage ;; + *) + usage + exit 1 + ;; + esac +} + +main "$@" diff --git a/scripts/start_remote_ephemeral_mainnet.sh b/scripts/start_remote_ephemeral_mainnet.sh new file mode 100755 index 00000000000..ed9222e00a1 --- /dev/null +++ b/scripts/start_remote_ephemeral_mainnet.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash + +# Copyright © Aptos Foundation +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +PROGRAM=$(basename "$0") +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd -P) +DEFAULT_HOST="ubuntu@34.231.241.232" +DEFAULT_IDENTITY="$HOME/movement/mainnet-fork.pem" +DEFAULT_REMOTE_SCRIPT="/mnt/mainnet-volume/fork-run-20260722/bin/start_ephemeral_mainnet.sh" +DEFAULT_REMOTE_INCOMING="/mnt/mainnet-volume/ephemeral-mainnet/incoming" +DEFAULT_LOCAL_API_PORT=8080 +ALREADY_RUNNING_EXIT=3 +ARCHIVE_TO_CLEAN="" + +usage() { + cat < through SSH. Validator P2P and REST listeners +are never bound to the host's public interface. +EOF +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "required command not found: $1" +} + +cleanup() { + if [[ -n "$ARCHIVE_TO_CLEAN" ]]; then + rm -f "$ARCHIVE_TO_CLEAN" + fi +} + +run_preflight() { + local destination=$1 + local identity=$2 + local remote_script=$3 + local status + + set +e + ssh -i "$identity" "$destination" "$remote_script" preflight + status=$? + set -e + if ((status == ALREADY_RUNNING_EXIT)); then + exit "$ALREADY_RUNNING_EXIT" + fi + ((status == 0)) || die "remote preflight failed with exit code $status" +} + +main() { + local requested_sha=${1:-} + [[ -n "$requested_sha" ]] || { + usage + exit 1 + } + if [[ "$requested_sha" == "-h" || "$requested_sha" == "--help" ]]; then + usage + exit 0 + fi + shift + + local destination=$DEFAULT_HOST + local identity=$DEFAULT_IDENTITY + local remote_script=$DEFAULT_REMOTE_SCRIPT + local remote_incoming=$DEFAULT_REMOTE_INCOMING + local local_api_port=$DEFAULT_LOCAL_API_PORT + while (($#)); do + case "$1" in + --host) + [[ $# -ge 2 ]] || die "--host requires a value" + destination=$2 + shift 2 + ;; + --identity) + [[ $# -ge 2 ]] || die "--identity requires a value" + identity=$2 + shift 2 + ;; + --remote-script) + [[ $# -ge 2 ]] || die "--remote-script requires a value" + remote_script=$2 + shift 2 + ;; + --remote-incoming) + [[ $# -ge 2 ]] || die "--remote-incoming requires a value" + remote_incoming=$2 + shift 2 + ;; + --local-api-port) + [[ $# -ge 2 ]] || die "--local-api-port requires a value" + local_api_port=$2 + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + *) + die "unknown option: $1" + ;; + esac + done + + [[ "$requested_sha" =~ ^[0-9a-fA-F]{7,40}$ ]] || + die "GITHUB_SHA must contain 7 to 40 hexadecimal characters" + if ! [[ "$local_api_port" =~ ^[1-9][0-9]*$ ]] || + ((local_api_port > 65535)); then + die "--local-api-port must be between 1 and 65535" + fi + [[ -f "$identity" ]] || die "SSH identity file does not exist: $identity" + require_command git + require_command gzip + require_command mktemp + require_command scp + require_command ssh + + git -C "$REPO_ROOT" fetch origin m1 + local full_sha + full_sha=$(git -C "$REPO_ROOT" rev-parse --verify "${requested_sha}^{commit}") || + die "commit does not exist after fetching origin/m1: $requested_sha" + git -C "$REPO_ROOT" merge-base --is-ancestor "$full_sha" origin/m1 || + die "commit $full_sha is not contained in origin/m1" + + run_preflight "$destination" "$identity" "$remote_script" + + local archive + archive=$(mktemp "${TMPDIR:-/tmp}/ephemeral-mainnet.${full_sha}.XXXXXX.tar.gz") + ARCHIVE_TO_CLEAN=$archive + trap cleanup EXIT + echo "Archiving m1 commit $full_sha..." + git -C "$REPO_ROOT" archive --format=tar "$full_sha" | gzip -1 >"$archive" + + local remote_archive="$remote_incoming/$full_sha.tar.gz" + echo "Uploading source archive to $destination..." + ssh -i "$identity" "$destination" mkdir -p "$remote_incoming" + scp -i "$identity" "$archive" "$destination:$remote_archive.uploading" + ssh -i "$identity" "$destination" \ + mv "$remote_archive.uploading" "$remote_archive" + + echo "Starting remote build and fork..." + set +e + ssh -i "$identity" "$destination" \ + "$remote_script" start "$full_sha" "$remote_archive" + local status=$? + set -e + if ((status == ALREADY_RUNNING_EXIT)); then + exit "$ALREADY_RUNNING_EXIT" + fi + ((status == 0)) || die "remote launcher failed with exit code $status" + + local remote_api + remote_api=$(ssh -i "$identity" "$destination" "$remote_script" endpoint) + [[ "$remote_api" =~ ^127\.0\.0\.1:[1-9][0-9]*$ ]] || + die "remote launcher returned an invalid loopback API endpoint: $remote_api" + + local key_dir="${XDG_CACHE_HOME:-$HOME/.cache}/aptos-ephemeral-mainnet" + local key_file="$key_dir/$full_sha.key" + mkdir -p "$key_dir" + ( + umask 077 + ssh -i "$identity" "$destination" "$remote_script" test-key >"$key_file" + ) + [[ -s "$key_file" ]] || die "remote launcher returned an empty test key" + + local control_socket="/tmp/aptos-mainnet-${USER:-user}-${full_sha:0:12}.sock" + if [[ -S "$control_socket" ]]; then + if ssh -S "$control_socket" -O check "$destination" >/dev/null 2>&1; then + die "an SSH tunnel already uses control socket $control_socket" + fi + rm -f "$control_socket" + fi + ssh -i "$identity" \ + -M -S "$control_socket" -fN \ + -o ExitOnForwardFailure=yes \ + -L "127.0.0.1:$local_api_port:$remote_api" \ + "$destination" + + echo + echo "Submission endpoint: http://127.0.0.1:$local_api_port" + echo "Local test key: $key_file" + echo "Validator P2P and REST remain loopback-only on 34.231.241.232." + echo + echo "Contract publishing pattern:" + echo " movement move publish --package-dir \\" + echo " --sender-account 0x573537299646e0dfab6ca81086edccf73f77b841f30dde6bfbd730ed428479bf \\" + echo " --private-key-file $key_file \\" + echo " --url http://127.0.0.1:$local_api_port --assume-yes" + echo + echo "Close API tunnel:" + echo " ssh -S $control_socket -O exit $destination" +} + +main "$@" diff --git a/scripts/tests/remote_ephemeral_mainnet_test.sh b/scripts/tests/remote_ephemeral_mainnet_test.sh new file mode 100755 index 00000000000..1a06694a984 --- /dev/null +++ b/scripts/tests/remote_ephemeral_mainnet_test.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# Copyright © Aptos Foundation +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/../.." && pwd -P) +LAUNCHER="$ROOT/scripts/remote/start_ephemeral_mainnet.sh" +TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/remote-ephemeral-mainnet-test.XXXXXX") + +cleanup() { + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +mkdir -p \ + "$TEST_ROOT/repo" \ + "$TEST_ROOT/source-db" \ + "$TEST_ROOT/run/cargo/bin" \ + "$TEST_ROOT/run/rustup" + +git -C "$TEST_ROOT/repo" init -q +git -C "$TEST_ROOT/repo" config user.name "Ephemeral Fork Test" +git -C "$TEST_ROOT/repo" config user.email "ephemeral-fork@example.com" +printf '%s\n' "test source" >"$TEST_ROOT/repo/source.txt" +git -C "$TEST_ROOT/repo" add source.txt +git -C "$TEST_ROOT/repo" commit -q -m "test source" +SHA=$(git -C "$TEST_ROOT/repo" rev-parse HEAD) +git -C "$TEST_ROOT/repo" archive --format=tar "$SHA" | + gzip -1 >"$TEST_ROOT/source.tar.gz" + +mkdir -p "$TEST_ROOT/service/builds/$SHA/target/release" +ln -s /usr/bin/true "$TEST_ROOT/service/builds/$SHA/target/release/aptos-node" + +# macOS does not provide flock; the remote Linux host does. The no-op shim is +# sufficient because this test invokes one launcher process at a time. +if ! command -v flock >/dev/null 2>&1; then + ln -s /usr/bin/true "$TEST_ROOT/run/cargo/bin/flock" +fi + +launcher_env=( + "EPHEMERAL_RUN_ROOT=$TEST_ROOT/run" + "EPHEMERAL_SERVICE_ROOT=$TEST_ROOT/service" + "EPHEMERAL_SOURCE_DB=$TEST_ROOT/source-db" + "EPHEMERAL_FORK_TOOL=/usr/bin/true" + "EPHEMERAL_FORK_LAUNCHER=/usr/bin/true" + "CARGO_HOME=$TEST_ROOT/run/cargo" + "RUSTUP_HOME=$TEST_ROOT/run/rustup" +) + +env "${launcher_env[@]}" "$LAUNCHER" start "$SHA" "$TEST_ROOT/source.tar.gz" +[[ $(cat "$TEST_ROOT/service/active/sha") == "$SHA" ]] +[[ -n $(cat "$TEST_ROOT/service/active/work-dir") ]] + +set +e +env \ + "${launcher_env[@]}" \ + "EPHEMERAL_SERVICE_ROOT=$TEST_ROOT/service-mismatch" \ + "$LAUNCHER" start \ + 0000000000000000000000000000000000000000 \ + "$TEST_ROOT/source.tar.gz" >/dev/null 2>&1 +mismatch_status=$? +set -e +[[ "$mismatch_status" -ne 0 ]] + +echo "remote ephemeral mainnet launcher test passed" From 07cbe29951129185bbb1074347bef57ac3705eac Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 27 Jul 2026 15:30:48 -0700 Subject: [PATCH 11/21] Expose one managed fork submission API --- scripts/remote/start_ephemeral_mainnet.sh | 140 +++++++++++++++++- scripts/remote/tcp_forward.py | 63 ++++++++ scripts/start_remote_ephemeral_mainnet.sh | 52 ++----- .../tests/remote_ephemeral_mainnet_test.sh | 1 + 4 files changed, 215 insertions(+), 41 deletions(-) create mode 100755 scripts/remote/tcp_forward.py diff --git a/scripts/remote/start_ephemeral_mainnet.sh b/scripts/remote/start_ephemeral_mainnet.sh index 3c37252675a..f7554cbd780 100755 --- a/scripts/remote/start_ephemeral_mainnet.sh +++ b/scripts/remote/start_ephemeral_mainnet.sh @@ -11,12 +11,15 @@ SERVICE_ROOT=${EPHEMERAL_SERVICE_ROOT:-/mnt/mainnet-volume/ephemeral-mainnet} SOURCE_DB=${EPHEMERAL_SOURCE_DB:-$RUN_ROOT/staging-db-v5} FORK_TOOL=${EPHEMERAL_FORK_TOOL:-$RUN_ROOT/target/release/aptos-debugger} FORK_LAUNCHER=${EPHEMERAL_FORK_LAUNCHER:-$RUN_ROOT/bin/ephemeral_fork.sh} +TCP_FORWARDER=${EPHEMERAL_TCP_FORWARDER:-$RUN_ROOT/bin/tcp_forward.py} CARGO_HOME=${CARGO_HOME:-$RUN_ROOT/cargo} RUSTUP_HOME=${RUSTUP_HOME:-$RUN_ROOT/rustup} TEST_ACCOUNT=${EPHEMERAL_TEST_ACCOUNT:-0x573537299646e0dfab6ca81086edccf73f77b841f30dde6bfbd730ed428479bf} VALIDATORS=${EPHEMERAL_VALIDATORS:-2} CHAIN_ID=${EPHEMERAL_CHAIN_ID:-42} HEALTH_TIMEOUT=${EPHEMERAL_HEALTH_TIMEOUT:-180} +PUBLIC_API_PORT=${EPHEMERAL_PUBLIC_API_PORT:-8080} +PUBLIC_API_ENABLED=${EPHEMERAL_PUBLIC_API_ENABLED:-true} ALREADY_RUNNING_EXIT=3 export PATH="$CARGO_HOME/bin:$PATH" @@ -29,6 +32,8 @@ Usage: $PROGRAM start FULL_GITHUB_SHA SOURCE_ARCHIVE.tar.gz $PROGRAM endpoint $PROGRAM test-key + $PROGRAM expose-api [PUBLIC_PORT] + $PROGRAM stop-api This command is intended to be invoked by scripts/start_remote_ephemeral_mainnet.sh. It refuses to build or start while any aptos-node process is already running. @@ -75,6 +80,9 @@ report_running_network() { fi echo "An ephemeral network is already running; refusing to replace it." echo " Recorded SHA: $active_sha" + if public_api_running; then + echo " Public submission endpoint: http://34.231.241.232:$(cat "$SERVICE_ROOT/public-api/port")" + fi local pid command config api response while IFS= read -r pid; do @@ -109,8 +117,17 @@ print( return 0 } +public_api_running() { + local pid_file="$SERVICE_ROOT/public-api/pid" + [[ -f "$pid_file" ]] || return 1 + local pid + pid=$(cat "$pid_file") + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || return 1 + kill -0 "$pid" 2>/dev/null +} + active_api_endpoint() { - local pids pid command config api + local pids pid command config api candidate="" pids=$(pgrep -x aptos-node || true) [[ -n "$pids" ]] || die "no aptos-node process is running" while IFS= read -r pid; do @@ -119,12 +136,13 @@ active_api_endpoint() { if [[ -n "$config" && -f "$config" ]]; then api=$(api_address_from_config "$config") if [[ "$api" == 127.0.0.1:* ]]; then - printf '%s\n' "$api" - return + candidate=$api fi fi done <<<"$pids" - die "running validators do not expose a loopback REST endpoint" + [[ -n "$candidate" ]] || + die "running validators do not expose a loopback REST endpoint" + printf '%s\n' "$candidate" } active_test_key() { @@ -137,6 +155,111 @@ active_test_key() { cat "$key_file" } +expose_public_api_locked() { + local public_port=${1:-$PUBLIC_API_PORT} + if ! [[ "$public_port" =~ ^[1-9][0-9]*$ ]] || + ((public_port > 65535)); then + die "public API port must be between 1 and 65535" + fi + [[ -x "$TCP_FORWARDER" ]] || die "TCP forwarder is missing: $TCP_FORWARDER" + require_command ss + + local target target_port + target=$(active_api_endpoint) + target_port=${target##*:} + if public_api_running; then + local existing_port existing_target + existing_port=$(cat "$SERVICE_ROOT/public-api/port") + existing_target=$(cat "$SERVICE_ROOT/public-api/target") + if [[ "$existing_port" == "$public_port" && "$existing_target" == "$target" ]]; then + echo "Public submission endpoint is already running at http://34.231.241.232:$public_port" + return + fi + die "a managed public API proxy is already running for $existing_target on port $existing_port" + fi + + if ss -ltnH "sport = :$public_port" | grep -q .; then + die "TCP port $public_port is already in use" + fi + + local proxy_dir="$SERVICE_ROOT/public-api" + mkdir -p "$proxy_dir" + nohup "$TCP_FORWARDER" \ + --listen-host 0.0.0.0 \ + --listen-port "$public_port" \ + --target-host 127.0.0.1 \ + --target-port "$target_port" \ + >"$proxy_dir/proxy.log" 2>&1 &- & + local proxy_pid + proxy_pid=$! + printf '%s\n' "$proxy_pid" >"$proxy_dir/pid" + printf '%s\n' "$public_port" >"$proxy_dir/port" + printf '%s\n' "$target" >"$proxy_dir/target" + + local attempt + for ((attempt = 0; attempt < 30; attempt++)); do + if ! kill -0 "$proxy_pid" 2>/dev/null; then + tail -n 80 "$proxy_dir/proxy.log" >&2 || true + die "public API proxy exited during startup" + fi + if curl --fail --silent --max-time 2 "http://127.0.0.1:$public_port/v1/" >/dev/null 2>&1; then + echo "Public submission endpoint: http://34.231.241.232:$public_port" + return + fi + sleep 1 + done + die "public API proxy did not become healthy" +} + +expose_public_api() { + mkdir -p "$SERVICE_ROOT" + exec 9>"$SERVICE_ROOT/launcher.lock" + flock 9 + expose_public_api_locked "$@" +} + +stop_public_api_locked() { + local pid_file="$SERVICE_ROOT/public-api/pid" + if [[ ! -f "$pid_file" ]]; then + echo "No managed public API proxy is recorded." + return + fi + + local pid + pid=$(cat "$pid_file") + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || die "invalid public API proxy PID: $pid" + if ! kill -0 "$pid" 2>/dev/null; then + echo "Managed public API proxy PID $pid is already stopped." + rm -f "$pid_file" + return + fi + + local command + command=$(ps -p "$pid" -o command=) + case "$command" in + *"$TCP_FORWARDER"*) ;; + *) die "refusing to stop PID $pid because it is not the managed TCP forwarder" ;; + esac + + kill "$pid" + local attempt + for ((attempt = 0; attempt < 50; attempt++)); do + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + done + kill -0 "$pid" 2>/dev/null && + die "public API proxy PID $pid did not stop after SIGTERM" + rm -f "$pid_file" + echo "Stopped managed public API proxy PID $pid." +} + +stop_public_api() { + mkdir -p "$SERVICE_ROOT" + exec 9>"$SERVICE_ROOT/launcher.lock" + flock 9 + stop_public_api_locked +} + preflight() { mkdir -p "$SERVICE_ROOT" exec 9>"$SERVICE_ROOT/launcher.lock" @@ -200,6 +323,7 @@ build_node() { start_network() { local sha=${1:-} local archive=${2:-} + local public_api_port=${3:-$PUBLIC_API_PORT} [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || die "start requires a full lowercase GitHub SHA" [[ -n "$archive" ]] || die "start requires a source archive path" @@ -240,11 +364,15 @@ start_network() { --validators "$VALIDATORS" \ --chain-id "$CHAIN_ID" \ --health-timeout "$HEALTH_TIMEOUT" \ - --work-dir "$work_dir" + --work-dir "$work_dir" \ + 9>&- mkdir -p "$SERVICE_ROOT/active" printf '%s\n' "$sha" >"$SERVICE_ROOT/active/sha" printf '%s\n' "$work_dir" >"$SERVICE_ROOT/active/work-dir" + if [[ "$PUBLIC_API_ENABLED" == true ]]; then + expose_public_api_locked "$public_api_port" + fi echo "Remote ephemeral mainnet started from m1 commit $sha." } @@ -256,6 +384,8 @@ main() { start) start_network "$@" ;; endpoint) active_api_endpoint "$@" ;; test-key) active_test_key "$@" ;; + expose-api) expose_public_api "$@" ;; + stop-api) stop_public_api "$@" ;; -h | --help | help) usage ;; *) usage diff --git a/scripts/remote/tcp_forward.py b/scripts/remote/tcp_forward.py new file mode 100755 index 00000000000..a582d769fe4 --- /dev/null +++ b/scripts/remote/tcp_forward.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Copyright © Aptos Foundation +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import select +import socket +import socketserver + + +class ForwardHandler(socketserver.BaseRequestHandler): + target: tuple[str, int] + + def handle(self) -> None: + with socket.create_connection(self.target, timeout=10) as upstream: + peers = { + self.request: upstream, + upstream: self.request, + } + while True: + readable, _, _ = select.select(peers, [], [], 30) + if not readable: + continue + for source in readable: + data = source.recv(64 * 1024) + if not data: + return + peers[source].sendall(data) + + +class ForwardServer(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Forward one TCP listener to a loopback service." + ) + parser.add_argument("--listen-host", default="0.0.0.0") + parser.add_argument("--listen-port", type=int, required=True) + parser.add_argument("--target-host", default="127.0.0.1") + parser.add_argument("--target-port", type=int, required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + for name, port in ( + ("listen port", args.listen_port), + ("target port", args.target_port), + ): + if not 1 <= port <= 65535: + raise ValueError(f"{name} must be between 1 and 65535") + + ForwardHandler.target = (args.target_host, args.target_port) + with ForwardServer((args.listen_host, args.listen_port), ForwardHandler) as server: + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/start_remote_ephemeral_mainnet.sh b/scripts/start_remote_ephemeral_mainnet.sh index ed9222e00a1..e1cd06e788e 100755 --- a/scripts/start_remote_ephemeral_mainnet.sh +++ b/scripts/start_remote_ephemeral_mainnet.sh @@ -11,7 +11,7 @@ DEFAULT_HOST="ubuntu@34.231.241.232" DEFAULT_IDENTITY="$HOME/movement/mainnet-fork.pem" DEFAULT_REMOTE_SCRIPT="/mnt/mainnet-volume/fork-run-20260722/bin/start_ephemeral_mainnet.sh" DEFAULT_REMOTE_INCOMING="/mnt/mainnet-volume/ephemeral-mainnet/incoming" -DEFAULT_LOCAL_API_PORT=8080 +DEFAULT_PUBLIC_API_PORT=8080 ALREADY_RUNNING_EXIT=3 ARCHIVE_TO_CLEAN="" @@ -27,15 +27,15 @@ Options: --identity PATH SSH private key. Default: $DEFAULT_IDENTITY --remote-script PATH Installed host launcher. Default: $DEFAULT_REMOTE_SCRIPT --remote-incoming PATH Archive upload directory. Default: $DEFAULT_REMOTE_INCOMING - --local-api-port PORT Local submission endpoint port. Default: $DEFAULT_LOCAL_API_PORT + --public-api-port PORT Public submission endpoint port. Default: $DEFAULT_PUBLIC_API_PORT The SHA must resolve to a commit contained in origin/m1. If an aptos-node process is already running on the host, the command reports it and exits without uploading source, building, stopping, or replacing anything. After a successful start, one remote loopback REST endpoint is forwarded to -http://127.0.0.1: through SSH. Validator P2P and REST listeners -are never bound to the host's public interface. +http://34.231.241.232:. Validator P2P and native REST listeners +remain loopback-only; only the managed submission proxy binds publicly. EOF } @@ -86,7 +86,7 @@ main() { local identity=$DEFAULT_IDENTITY local remote_script=$DEFAULT_REMOTE_SCRIPT local remote_incoming=$DEFAULT_REMOTE_INCOMING - local local_api_port=$DEFAULT_LOCAL_API_PORT + local public_api_port=$DEFAULT_PUBLIC_API_PORT while (($#)); do case "$1" in --host) @@ -109,9 +109,9 @@ main() { remote_incoming=$2 shift 2 ;; - --local-api-port) - [[ $# -ge 2 ]] || die "--local-api-port requires a value" - local_api_port=$2 + --public-api-port) + [[ $# -ge 2 ]] || die "--public-api-port requires a value" + public_api_port=$2 shift 2 ;; -h | --help) @@ -126,9 +126,9 @@ main() { [[ "$requested_sha" =~ ^[0-9a-fA-F]{7,40}$ ]] || die "GITHUB_SHA must contain 7 to 40 hexadecimal characters" - if ! [[ "$local_api_port" =~ ^[1-9][0-9]*$ ]] || - ((local_api_port > 65535)); then - die "--local-api-port must be between 1 and 65535" + if ! [[ "$public_api_port" =~ ^[1-9][0-9]*$ ]] || + ((public_api_port > 65535)); then + die "--public-api-port must be between 1 and 65535" fi [[ -f "$identity" ]] || die "SSH identity file does not exist: $identity" require_command git @@ -163,7 +163,7 @@ main() { echo "Starting remote build and fork..." set +e ssh -i "$identity" "$destination" \ - "$remote_script" start "$full_sha" "$remote_archive" + "$remote_script" start "$full_sha" "$remote_archive" "$public_api_port" local status=$? set -e if ((status == ALREADY_RUNNING_EXIT)); then @@ -171,11 +171,6 @@ main() { fi ((status == 0)) || die "remote launcher failed with exit code $status" - local remote_api - remote_api=$(ssh -i "$identity" "$destination" "$remote_script" endpoint) - [[ "$remote_api" =~ ^127\.0\.0\.1:[1-9][0-9]*$ ]] || - die "remote launcher returned an invalid loopback API endpoint: $remote_api" - local key_dir="${XDG_CACHE_HOME:-$HOME/.cache}/aptos-ephemeral-mainnet" local key_file="$key_dir/$full_sha.key" mkdir -p "$key_dir" @@ -185,32 +180,17 @@ main() { ) [[ -s "$key_file" ]] || die "remote launcher returned an empty test key" - local control_socket="/tmp/aptos-mainnet-${USER:-user}-${full_sha:0:12}.sock" - if [[ -S "$control_socket" ]]; then - if ssh -S "$control_socket" -O check "$destination" >/dev/null 2>&1; then - die "an SSH tunnel already uses control socket $control_socket" - fi - rm -f "$control_socket" - fi - ssh -i "$identity" \ - -M -S "$control_socket" -fN \ - -o ExitOnForwardFailure=yes \ - -L "127.0.0.1:$local_api_port:$remote_api" \ - "$destination" - + local public_host=${destination#*@} echo - echo "Submission endpoint: http://127.0.0.1:$local_api_port" + echo "Submission endpoint: http://$public_host:$public_api_port" echo "Local test key: $key_file" - echo "Validator P2P and REST remain loopback-only on 34.231.241.232." + echo "Validator P2P and native REST remain loopback-only." echo echo "Contract publishing pattern:" echo " movement move publish --package-dir \\" echo " --sender-account 0x573537299646e0dfab6ca81086edccf73f77b841f30dde6bfbd730ed428479bf \\" echo " --private-key-file $key_file \\" - echo " --url http://127.0.0.1:$local_api_port --assume-yes" - echo - echo "Close API tunnel:" - echo " ssh -S $control_socket -O exit $destination" + echo " --url http://$public_host:$public_api_port --assume-yes" } main "$@" diff --git a/scripts/tests/remote_ephemeral_mainnet_test.sh b/scripts/tests/remote_ephemeral_mainnet_test.sh index 1a06694a984..42324f26907 100755 --- a/scripts/tests/remote_ephemeral_mainnet_test.sh +++ b/scripts/tests/remote_ephemeral_mainnet_test.sh @@ -45,6 +45,7 @@ launcher_env=( "EPHEMERAL_SOURCE_DB=$TEST_ROOT/source-db" "EPHEMERAL_FORK_TOOL=/usr/bin/true" "EPHEMERAL_FORK_LAUNCHER=/usr/bin/true" + "EPHEMERAL_PUBLIC_API_ENABLED=false" "CARGO_HOME=$TEST_ROOT/run/cargo" "RUSTUP_HOME=$TEST_ROOT/run/rustup" ) From 135a4aa9d3fd9413e143a9512dd588c5df41161c Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 27 Jul 2026 16:00:31 -0700 Subject: [PATCH 12/21] Move fork harness under testsuite --- {scripts => testsuite/forktest}/ephemeral_fork.sh | 0 .../forktest}/remote/start_ephemeral_mainnet.sh | 3 ++- {scripts => testsuite/forktest}/remote/tcp_forward.py | 0 .../forktest}/start_remote_ephemeral_mainnet.sh | 2 +- {scripts => testsuite/forktest}/tests/ephemeral_fork_test.sh | 4 ++-- .../forktest}/tests/remote_ephemeral_mainnet_test.sh | 4 ++-- 6 files changed, 7 insertions(+), 6 deletions(-) rename {scripts => testsuite/forktest}/ephemeral_fork.sh (100%) rename {scripts => testsuite/forktest}/remote/start_ephemeral_mainnet.sh (99%) rename {scripts => testsuite/forktest}/remote/tcp_forward.py (100%) rename {scripts => testsuite/forktest}/start_remote_ephemeral_mainnet.sh (99%) rename {scripts => testsuite/forktest}/tests/ephemeral_fork_test.sh (98%) rename {scripts => testsuite/forktest}/tests/remote_ephemeral_mainnet_test.sh (94%) diff --git a/scripts/ephemeral_fork.sh b/testsuite/forktest/ephemeral_fork.sh similarity index 100% rename from scripts/ephemeral_fork.sh rename to testsuite/forktest/ephemeral_fork.sh diff --git a/scripts/remote/start_ephemeral_mainnet.sh b/testsuite/forktest/remote/start_ephemeral_mainnet.sh similarity index 99% rename from scripts/remote/start_ephemeral_mainnet.sh rename to testsuite/forktest/remote/start_ephemeral_mainnet.sh index f7554cbd780..6d8e5fb9e42 100755 --- a/scripts/remote/start_ephemeral_mainnet.sh +++ b/testsuite/forktest/remote/start_ephemeral_mainnet.sh @@ -35,7 +35,8 @@ Usage: $PROGRAM expose-api [PUBLIC_PORT] $PROGRAM stop-api -This command is intended to be invoked by scripts/start_remote_ephemeral_mainnet.sh. +This command is intended to be invoked by +testsuite/forktest/start_remote_ephemeral_mainnet.sh. It refuses to build or start while any aptos-node process is already running. EOF } diff --git a/scripts/remote/tcp_forward.py b/testsuite/forktest/remote/tcp_forward.py similarity index 100% rename from scripts/remote/tcp_forward.py rename to testsuite/forktest/remote/tcp_forward.py diff --git a/scripts/start_remote_ephemeral_mainnet.sh b/testsuite/forktest/start_remote_ephemeral_mainnet.sh similarity index 99% rename from scripts/start_remote_ephemeral_mainnet.sh rename to testsuite/forktest/start_remote_ephemeral_mainnet.sh index e1cd06e788e..229d76e20ff 100755 --- a/scripts/start_remote_ephemeral_mainnet.sh +++ b/testsuite/forktest/start_remote_ephemeral_mainnet.sh @@ -6,7 +6,7 @@ set -euo pipefail PROGRAM=$(basename "$0") -REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd -P) +REPO_ROOT=$(cd "$(dirname "$0")/../.." && pwd -P) DEFAULT_HOST="ubuntu@34.231.241.232" DEFAULT_IDENTITY="$HOME/movement/mainnet-fork.pem" DEFAULT_REMOTE_SCRIPT="/mnt/mainnet-volume/fork-run-20260722/bin/start_ephemeral_mainnet.sh" diff --git a/scripts/tests/ephemeral_fork_test.sh b/testsuite/forktest/tests/ephemeral_fork_test.sh similarity index 98% rename from scripts/tests/ephemeral_fork_test.sh rename to testsuite/forktest/tests/ephemeral_fork_test.sh index 945a29a9582..2c03d49a0db 100755 --- a/scripts/tests/ephemeral_fork_test.sh +++ b/testsuite/forktest/tests/ephemeral_fork_test.sh @@ -5,8 +5,8 @@ set -euo pipefail -ROOT=$(cd "$(dirname "$0")/../.." && pwd -P) -LAUNCHER="$ROOT/scripts/ephemeral_fork.sh" +ROOT=$(cd "$(dirname "$0")/../../.." && pwd -P) +LAUNCHER="$ROOT/testsuite/forktest/ephemeral_fork.sh" TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/ephemeral-fork-test.XXXXXX") SOURCE_DB="$TEST_ROOT/source-db" WORK_DIR="$TEST_ROOT/work" diff --git a/scripts/tests/remote_ephemeral_mainnet_test.sh b/testsuite/forktest/tests/remote_ephemeral_mainnet_test.sh similarity index 94% rename from scripts/tests/remote_ephemeral_mainnet_test.sh rename to testsuite/forktest/tests/remote_ephemeral_mainnet_test.sh index 42324f26907..f3dac32c251 100755 --- a/scripts/tests/remote_ephemeral_mainnet_test.sh +++ b/testsuite/forktest/tests/remote_ephemeral_mainnet_test.sh @@ -5,8 +5,8 @@ set -euo pipefail -ROOT=$(cd "$(dirname "$0")/../.." && pwd -P) -LAUNCHER="$ROOT/scripts/remote/start_ephemeral_mainnet.sh" +ROOT=$(cd "$(dirname "$0")/../../.." && pwd -P) +LAUNCHER="$ROOT/testsuite/forktest/remote/start_ephemeral_mainnet.sh" TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/remote-ephemeral-mainnet-test.XXXXXX") cleanup() { From 90571e7ceee4aa90ed44d75aaa60b2525fe4d37f Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Mon, 27 Jul 2026 16:05:58 -0700 Subject: [PATCH 13/21] Hide internal fork lifecycle helper --- testsuite/forktest/{ => remote}/ephemeral_fork.sh | 0 testsuite/forktest/start_remote_ephemeral_mainnet.sh | 3 +++ testsuite/forktest/tests/ephemeral_fork_test.sh | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) rename testsuite/forktest/{ => remote}/ephemeral_fork.sh (100%) diff --git a/testsuite/forktest/ephemeral_fork.sh b/testsuite/forktest/remote/ephemeral_fork.sh similarity index 100% rename from testsuite/forktest/ephemeral_fork.sh rename to testsuite/forktest/remote/ephemeral_fork.sh diff --git a/testsuite/forktest/start_remote_ephemeral_mainnet.sh b/testsuite/forktest/start_remote_ephemeral_mainnet.sh index 229d76e20ff..493118b005f 100755 --- a/testsuite/forktest/start_remote_ephemeral_mainnet.sh +++ b/testsuite/forktest/start_remote_ephemeral_mainnet.sh @@ -22,6 +22,9 @@ Build and start an ephemeral mainnet fork on the configured remote host. Usage: $PROGRAM GITHUB_SHA [OPTIONS] +GITHUB_SHA is the only required argument. The options below override the +preconfigured test host when needed. + Options: --host USER@HOST SSH destination. Default: $DEFAULT_HOST --identity PATH SSH private key. Default: $DEFAULT_IDENTITY diff --git a/testsuite/forktest/tests/ephemeral_fork_test.sh b/testsuite/forktest/tests/ephemeral_fork_test.sh index 2c03d49a0db..5ea2af02bbf 100755 --- a/testsuite/forktest/tests/ephemeral_fork_test.sh +++ b/testsuite/forktest/tests/ephemeral_fork_test.sh @@ -6,7 +6,7 @@ set -euo pipefail ROOT=$(cd "$(dirname "$0")/../../.." && pwd -P) -LAUNCHER="$ROOT/testsuite/forktest/ephemeral_fork.sh" +LAUNCHER="$ROOT/testsuite/forktest/remote/ephemeral_fork.sh" TEST_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/ephemeral-fork-test.XXXXXX") SOURCE_DB="$TEST_ROOT/source-db" WORK_DIR="$TEST_ROOT/work" From a20621719657270c88328d4caf4cd5b721ea4654 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Tue, 28 Jul 2026 11:18:41 -0700 Subject: [PATCH 14/21] Update storage/db-tool/src/fork.rs Co-authored-by: Andy <17599867+ganymedio@users.noreply.github.com> --- storage/db-tool/src/fork.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index f3558b8445a..bf527fa05a9 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -379,8 +379,8 @@ fn parse_test_account_rekey( fn validate_requested_chain_id(chain_id: u8) -> Result { ensure!( - chain_id != 0 && chain_id != 1, - "fork chain ID must not be 0 or mainnet (1)", + chain_id != 0 && chain_id != 126, + "fork chain ID must not be 0 or mainnet (126)", ); Ok(ChainId::new(chain_id)) } From cccd45b5b7479b4bca04ecb866285fd9cad8ff3f Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Tue, 28 Jul 2026 11:22:57 -0700 Subject: [PATCH 15/21] Derive advertised fork endpoint host --- testsuite/forktest/remote/start_ephemeral_mainnet.sh | 6 +++--- testsuite/forktest/start_remote_ephemeral_mainnet.sh | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/testsuite/forktest/remote/start_ephemeral_mainnet.sh b/testsuite/forktest/remote/start_ephemeral_mainnet.sh index 6d8e5fb9e42..6d2d3a72519 100755 --- a/testsuite/forktest/remote/start_ephemeral_mainnet.sh +++ b/testsuite/forktest/remote/start_ephemeral_mainnet.sh @@ -82,7 +82,7 @@ report_running_network() { echo "An ephemeral network is already running; refusing to replace it." echo " Recorded SHA: $active_sha" if public_api_running; then - echo " Public submission endpoint: http://34.231.241.232:$(cat "$SERVICE_ROOT/public-api/port")" + echo " Public submission port: $(cat "$SERVICE_ROOT/public-api/port")" fi local pid command config api response @@ -173,7 +173,7 @@ expose_public_api_locked() { existing_port=$(cat "$SERVICE_ROOT/public-api/port") existing_target=$(cat "$SERVICE_ROOT/public-api/target") if [[ "$existing_port" == "$public_port" && "$existing_target" == "$target" ]]; then - echo "Public submission endpoint is already running at http://34.231.241.232:$public_port" + echo "Public submission proxy is already running on port $public_port" return fi die "a managed public API proxy is already running for $existing_target on port $existing_port" @@ -204,7 +204,7 @@ expose_public_api_locked() { die "public API proxy exited during startup" fi if curl --fail --silent --max-time 2 "http://127.0.0.1:$public_port/v1/" >/dev/null 2>&1; then - echo "Public submission endpoint: http://34.231.241.232:$public_port" + echo "Public submission proxy is healthy on port $public_port" return fi sleep 1 diff --git a/testsuite/forktest/start_remote_ephemeral_mainnet.sh b/testsuite/forktest/start_remote_ephemeral_mainnet.sh index 493118b005f..c5a2200754a 100755 --- a/testsuite/forktest/start_remote_ephemeral_mainnet.sh +++ b/testsuite/forktest/start_remote_ephemeral_mainnet.sh @@ -12,6 +12,7 @@ DEFAULT_IDENTITY="$HOME/movement/mainnet-fork.pem" DEFAULT_REMOTE_SCRIPT="/mnt/mainnet-volume/fork-run-20260722/bin/start_ephemeral_mainnet.sh" DEFAULT_REMOTE_INCOMING="/mnt/mainnet-volume/ephemeral-mainnet/incoming" DEFAULT_PUBLIC_API_PORT=8080 +DEFAULT_PUBLIC_HOST=${DEFAULT_HOST#*@} ALREADY_RUNNING_EXIT=3 ARCHIVE_TO_CLEAN="" @@ -37,7 +38,7 @@ process is already running on the host, the command reports it and exits without uploading source, building, stopping, or replacing anything. After a successful start, one remote loopback REST endpoint is forwarded to -http://34.231.241.232:. Validator P2P and native REST listeners +http://$DEFAULT_PUBLIC_HOST:. Validator P2P and native REST listeners remain loopback-only; only the managed submission proxy binds publicly. EOF } From 1590faf4a0587f81843286b6b58ec4cb5a250aec Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Tue, 28 Jul 2026 21:42:53 -0700 Subject: [PATCH 16/21] Reserve Movement mainnet chain ID in fork tests --- storage/db-tool/src/fork.rs | 3 ++- testsuite/forktest/remote/ephemeral_fork.sh | 4 ++-- testsuite/forktest/tests/ephemeral_fork_test.sh | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index bf527fa05a9..c2c6fdd54fa 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -971,8 +971,9 @@ mod tests { #[test] fn rejects_reserved_chain_ids() { assert!(validate_requested_chain_id(0).is_err()); - assert!(validate_requested_chain_id(1).is_err()); + assert_eq!(validate_requested_chain_id(1).unwrap().id(), 1); assert_eq!(validate_requested_chain_id(42).unwrap().id(), 42); + assert!(validate_requested_chain_id(126).is_err()); } #[test] diff --git a/testsuite/forktest/remote/ephemeral_fork.sh b/testsuite/forktest/remote/ephemeral_fork.sh index d36f6172037..ae57fcd118d 100755 --- a/testsuite/forktest/remote/ephemeral_fork.sh +++ b/testsuite/forktest/remote/ephemeral_fork.sh @@ -69,8 +69,8 @@ validate_positive_integer() { validate_chain_id() { local chain_id=$1 validate_positive_integer "--chain-id" "$chain_id" - ((chain_id > 1 && chain_id <= 255)) || - die "--chain-id must be between 2 and 255; 1 is reserved for mainnet" + ((chain_id <= 255 && chain_id != 126)) || + die "--chain-id must be between 1 and 255 and must not be Movement mainnet (126)" } validate_account_address() { diff --git a/testsuite/forktest/tests/ephemeral_fork_test.sh b/testsuite/forktest/tests/ephemeral_fork_test.sh index 5ea2af02bbf..f9bf873b0a6 100755 --- a/testsuite/forktest/tests/ephemeral_fork_test.sh +++ b/testsuite/forktest/tests/ephemeral_fork_test.sh @@ -194,9 +194,9 @@ if "$LAUNCHER" start \ --fork-tool "$MOCK_FORK_TOOL" \ --node-binary "$MOCK_NODE" \ --test-account-address 0x1234 \ - --chain-id 1 \ + --chain-id 126 \ --work-dir "$TEST_ROOT/mainnet-chain-id" >/dev/null 2>&1; then - echo "start unexpectedly accepted mainnet chain ID 1" >&2 + echo "start unexpectedly accepted Movement mainnet chain ID 126" >&2 exit 1 fi From 8dbed21e328f56527acd3ccd501f0751ae8c4d87 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 29 Jul 2026 11:10:21 -0700 Subject: [PATCH 17/21] Allow fork binaries from any origin branch --- .../remote/start_ephemeral_mainnet.sh | 4 ++-- .../start_remote_ephemeral_mainnet.sh | 23 ++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/testsuite/forktest/remote/start_ephemeral_mainnet.sh b/testsuite/forktest/remote/start_ephemeral_mainnet.sh index 6d2d3a72519..34bd524ff78 100755 --- a/testsuite/forktest/remote/start_ephemeral_mainnet.sh +++ b/testsuite/forktest/remote/start_ephemeral_mainnet.sh @@ -309,7 +309,7 @@ build_node() { mkdir -p "$source_dir" "$target_dir" tar -xzf "$archive" -C "$source_dir" - echo "Building aptos-node for m1 commit $sha..." >&2 + echo "Building aptos-node for origin commit $sha..." >&2 ( export CARGO_HOME RUSTUP_HOME export CARGO_TARGET_DIR="$target_dir" @@ -374,7 +374,7 @@ start_network() { if [[ "$PUBLIC_API_ENABLED" == true ]]; then expose_public_api_locked "$public_api_port" fi - echo "Remote ephemeral mainnet started from m1 commit $sha." + echo "Remote ephemeral mainnet started from origin commit $sha." } main() { diff --git a/testsuite/forktest/start_remote_ephemeral_mainnet.sh b/testsuite/forktest/start_remote_ephemeral_mainnet.sh index c5a2200754a..c88a0744fed 100755 --- a/testsuite/forktest/start_remote_ephemeral_mainnet.sh +++ b/testsuite/forktest/start_remote_ephemeral_mainnet.sh @@ -33,9 +33,9 @@ Options: --remote-incoming PATH Archive upload directory. Default: $DEFAULT_REMOTE_INCOMING --public-api-port PORT Public submission endpoint port. Default: $DEFAULT_PUBLIC_API_PORT -The SHA must resolve to a commit contained in origin/m1. If an aptos-node -process is already running on the host, the command reports it and exits -without uploading source, building, stopping, or replacing anything. +The SHA must resolve to a commit reachable from any branch in origin. If an +aptos-node process is already running on the host, the command reports it and +exits without uploading source, building, stopping, or replacing anything. After a successful start, one remote loopback REST endpoint is forwarded to http://$DEFAULT_PUBLIC_HOST:. Validator P2P and native REST listeners @@ -141,12 +141,19 @@ main() { require_command scp require_command ssh - git -C "$REPO_ROOT" fetch origin m1 + git -C "$REPO_ROOT" fetch --prune origin --no-tags local full_sha full_sha=$(git -C "$REPO_ROOT" rev-parse --verify "${requested_sha}^{commit}") || - die "commit does not exist after fetching origin/m1: $requested_sha" - git -C "$REPO_ROOT" merge-base --is-ancestor "$full_sha" origin/m1 || - die "commit $full_sha is not contained in origin/m1" + die "commit does not exist after fetching origin branches: $requested_sha" + local containing_refs + containing_refs=$( + git -C "$REPO_ROOT" for-each-ref \ + --contains "$full_sha" \ + --format='%(refname:short)' \ + refs/remotes/origin + ) + [[ -n "$containing_refs" ]] || + die "commit $full_sha is not reachable from any branch in origin" run_preflight "$destination" "$identity" "$remote_script" @@ -154,7 +161,7 @@ main() { archive=$(mktemp "${TMPDIR:-/tmp}/ephemeral-mainnet.${full_sha}.XXXXXX.tar.gz") ARCHIVE_TO_CLEAN=$archive trap cleanup EXIT - echo "Archiving m1 commit $full_sha..." + echo "Archiving origin commit $full_sha..." git -C "$REPO_ROOT" archive --format=tar "$full_sha" | gzip -1 >"$archive" local remote_archive="$remote_incoming/$full_sha.tar.gz" From 09c63ad57f302bface6e3404269573fa624606c9 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 29 Jul 2026 12:11:09 -0700 Subject: [PATCH 18/21] fix(forktest): normalize configs for older nodes --- testsuite/forktest/remote/ephemeral_fork.sh | 18 +++++++++++++++++- .../forktest/tests/ephemeral_fork_test.sh | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/testsuite/forktest/remote/ephemeral_fork.sh b/testsuite/forktest/remote/ephemeral_fork.sh index ae57fcd118d..8e87cf845a9 100755 --- a/testsuite/forktest/remote/ephemeral_fork.sh +++ b/testsuite/forktest/remote/ephemeral_fork.sh @@ -117,6 +117,21 @@ ensure_loopback_address() { esac } +normalize_candidate_config() { + local config=$1 + local normalized="$config.normalized" + + awk ' + /^api:$/ { in_api = 1 } + in_api && /^ simulation_filter:$/ { skip = 1; next } + skip && /^ / { next } + skip { skip = 0 } + in_api && /^[^ ]/ { in_api = 0 } + { print } + ' "$config" >"$normalized" + mv "$normalized" "$config" +} + ledger_summary() { python3 -c ' import json @@ -384,13 +399,14 @@ start_fork() { : >"$rest_urls_file" local start_complete=false - trap 'if [[ "$start_complete" != true ]]; then stop_nodes "$config_dir" true || true; fi' EXIT + trap 'if [[ "${start_complete:-false}" != true ]]; then stop_nodes "$config_dir" true || true; fi' EXIT trap 'exit 130' INT TERM local index config api_address rest_url for ((index = 0; index < validators; index++)); do config="$config_dir/$index/node.yaml" [[ -f "$config" ]] || die "missing generated node config: $config" + normalize_candidate_config "$config" api_address=$(api_address_from_config "$config") [[ -n "$api_address" ]] || die "could not read REST address from $config" ensure_loopback_address "$api_address" diff --git a/testsuite/forktest/tests/ephemeral_fork_test.sh b/testsuite/forktest/tests/ephemeral_fork_test.sh index f9bf873b0a6..962a645adce 100755 --- a/testsuite/forktest/tests/ephemeral_fork_test.sh +++ b/testsuite/forktest/tests/ephemeral_fork_test.sh @@ -89,6 +89,8 @@ for ((index = 0; index < validators; index++)); do api: enabled: true address: "127.0.0.1:$port" + simulation_filter: + rules: [] base: role: "validator" EOF @@ -171,6 +173,7 @@ grep -q "ledger_version=11" <<<"$status_output" expected_key="0x0000000000000000000000000000000000000000000000000000000000000abc" [[ $(cat "$WORK_DIR/configs/test-account-private-key") == "$expected_key" ]] +! grep -q simulation_filter "$WORK_DIR/configs/0/node.yaml" "$LAUNCHER" stop --work-dir "$WORK_DIR" >/dev/null if "$LAUNCHER" status --work-dir "$WORK_DIR" >/dev/null 2>&1; then From f2c80fa9993acc71e74c5a260b97bf29715e2c36 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 29 Jul 2026 12:12:35 -0700 Subject: [PATCH 19/21] fix(forktest): preserve API normalization scope --- testsuite/forktest/remote/ephemeral_fork.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/forktest/remote/ephemeral_fork.sh b/testsuite/forktest/remote/ephemeral_fork.sh index 8e87cf845a9..6602b10ff15 100755 --- a/testsuite/forktest/remote/ephemeral_fork.sh +++ b/testsuite/forktest/remote/ephemeral_fork.sh @@ -122,7 +122,7 @@ normalize_candidate_config() { local normalized="$config.normalized" awk ' - /^api:$/ { in_api = 1 } + /^api:$/ { in_api = 1; print; next } in_api && /^ simulation_filter:$/ { skip = 1; next } skip && /^ / { next } skip { skip = 0 } @@ -399,7 +399,7 @@ start_fork() { : >"$rest_urls_file" local start_complete=false - trap 'if [[ "${start_complete:-false}" != true ]]; then stop_nodes "$config_dir" true || true; fi' EXIT + trap 'if [[ "${start_complete:-false}" != true && -n "${config_dir:-}" ]]; then stop_nodes "$config_dir" true || true; fi' EXIT trap 'exit 130' INT TERM local index config api_address rest_url From b5f6f45b505da9378bced68e1288aebe86f9cf06 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 29 Jul 2026 12:14:34 -0700 Subject: [PATCH 20/21] fix(forktest): bridge candidate config schema drift --- testsuite/forktest/remote/ephemeral_fork.sh | 31 ++++++++++++++++--- .../forktest/tests/ephemeral_fork_test.sh | 11 ++++++- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/testsuite/forktest/remote/ephemeral_fork.sh b/testsuite/forktest/remote/ephemeral_fork.sh index 6602b10ff15..50ba547cc65 100755 --- a/testsuite/forktest/remote/ephemeral_fork.sh +++ b/testsuite/forktest/remote/ephemeral_fork.sh @@ -122,11 +122,32 @@ normalize_candidate_config() { local normalized="$config.normalized" awk ' - /^api:$/ { in_api = 1; print; next } - in_api && /^ simulation_filter:$/ { skip = 1; next } - skip && /^ / { next } - skip { skip = 0 } - in_api && /^[^ ]/ { in_api = 0 } + function incompatible(section, key) { + return \ + (section == "api" && key == "simulation_filter") || \ + (section == "consensus" && (key == "channel_size" || key == "enable_pipeline")) || \ + (section == "consensus_observer" && key == "enable_pipeline") || \ + (section == "execution" && key == "transaction_filter") + } + + skip { + if (/^ / || /^[[:space:]]*$/) { + next + } + skip = 0 + } + /^[^ ]/ { + section = $1 + sub(/:$/, "", section) + } + /^ [^ ]/ { + key = $1 + sub(/:$/, "", key) + if (incompatible(section, key)) { + skip = 1 + next + } + } { print } ' "$config" >"$normalized" mv "$normalized" "$config" diff --git a/testsuite/forktest/tests/ephemeral_fork_test.sh b/testsuite/forktest/tests/ephemeral_fork_test.sh index 962a645adce..b5c85eecbb1 100755 --- a/testsuite/forktest/tests/ephemeral_fork_test.sh +++ b/testsuite/forktest/tests/ephemeral_fork_test.sh @@ -91,6 +91,14 @@ api: address: "127.0.0.1:$port" simulation_filter: rules: [] +consensus: + channel_size: 30 + enable_pipeline: true +consensus_observer: + enable_pipeline: true +execution: + transaction_filter: + rules: [] base: role: "validator" EOF @@ -173,7 +181,8 @@ grep -q "ledger_version=11" <<<"$status_output" expected_key="0x0000000000000000000000000000000000000000000000000000000000000abc" [[ $(cat "$WORK_DIR/configs/test-account-private-key") == "$expected_key" ]] -! grep -q simulation_filter "$WORK_DIR/configs/0/node.yaml" +! grep -Eq "simulation_filter|channel_size|enable_pipeline|transaction_filter" \ + "$WORK_DIR/configs/0/node.yaml" "$LAUNCHER" stop --work-dir "$WORK_DIR" >/dev/null if "$LAUNCHER" status --work-dir "$WORK_DIR" >/dev/null 2>&1; then From cab3ccf3b094212d826ce524b4f22b1f108c5483 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 29 Jul 2026 15:39:13 -0700 Subject: [PATCH 21/21] fix(fork): disable pruning in ephemeral validators --- storage/db-tool/src/fork.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/db-tool/src/fork.rs b/storage/db-tool/src/fork.rs index c2c6fdd54fa..89e6ee2f23d 100644 --- a/storage/db-tool/src/fork.rs +++ b/storage/db-tool/src/fork.rs @@ -801,6 +801,7 @@ fn write_fork_configs( ) })?; config.storage.dir = validator_db_dir; + config.storage.storage_pruner_config = NO_OP_STORAGE_PRUNER_CONFIG; config.base.waypoint = WaypointConfig::FromConfig(waypoint); config.execution.genesis = Some(fork_txn.clone()); config.execution.genesis_waypoint = Some(WaypointConfig::FromConfig(waypoint)); @@ -1225,6 +1226,10 @@ mod tests { node_config.storage.dir().canonicalize().unwrap(), validator_db_dir.canonicalize().unwrap(), ); + assert_eq!( + node_config.storage.storage_pruner_config, + NO_OP_STORAGE_PRUNER_CONFIG, + ); assert!(node_config.api.address.ip().is_loopback()); assert!(node_config.indexer_grpc.address.ip().is_loopback()); assert!(!node_config.logger.enable_telemetry_remote_log);