From fca7589afcbc81cc810225d60ecddb17485c8a78 Mon Sep 17 00:00:00 2001 From: Benjamen Keroack Date: Wed, 7 Oct 2026 14:24:42 -0600 Subject: [PATCH 1/2] Index the genesis record a full -reindex finds in blk00000.dat (#931) (#932) A full -reindex empties the database, and ChainState::new then took the empty store for a new datadir: genesis was not in the block index, so it appended the genesis block to the last block file. Every reindex left a duplicate genesis record behind, and on a datadir switched from Bitcoin Core it wrote into one of Core's files. On an empty store, ChainState::new now reads the first record of blk00000.dat, through the xor.dat key, and when its payload is the genesis block byte for byte, indexes genesis there. satd and Core both write genesis first. A datadir whose files do not begin with it (a new one, a pruned one) gets it written as before. Core reaches the same result by finding genesis in the files during its reindex. Co-authored-by: Claude Opus 5.5 (1M context) (cherry picked from commit e17fe07b4d037ed1f0aa0900b315d75e800e8c97) --- node/src/chain/state.rs | 146 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 5 deletions(-) diff --git a/node/src/chain/state.rs b/node/src/chain/state.rs index 31c0edc2f..a7468d45f 100644 --- a/node/src/chain/state.rs +++ b/node/src/chain/state.rs @@ -974,6 +974,14 @@ impl ChainState { // are mode 644 satd:satd. Without this branch a validation // node sharing `blocks/` with a primary can never // `-reindex-chainstate`. + // + // A full `-reindex` keeps no block index, so the store is empty, but + // the block files still hold genesis as the first record of + // `blk00000.dat`, where satd and Core both write it (#931). Index it + // there: appending a second copy wrote to a block file, possibly one + // of Core's, on an operation that otherwise only reads them. Only a + // datadir whose files do not begin with genesis (a new one, a pruned + // one) gets it written. tracing::info!("Initializing chain with genesis block"); let flat_pos = if let Some(entry) = store.get_block_index(&genesis_hash) { @@ -986,6 +994,11 @@ impl ChainState { file_number: entry.file_number, data_pos: entry.data_pos, } + } else if let Some(pos) = genesis_record_on_disk(&mut flat_files, &genesis) { + tracing::info!( + "Genesis is the first record of blk00000.dat; indexing it there" + ); + pos } else { let block_data = serialize(&genesis); let pos = flat_files @@ -6358,11 +6371,11 @@ impl ChainState { if let Some(required) = short_of { // Where the stored chain picks up again past the hole, if it does: // blocks whose parent is nowhere in the files. The gap alone is - // not enough, because it can be gone by now. `ChainState::new` - // appended genesis to the empty database's block files before - // this scan, and when genesis does not fit in the file below a - // gap it rolls over into the missing file's number, so the scan - // reads past the hole. + // not enough, because it can be gone by now. When the block files + // do not begin with genesis, `ChainState::new` appended it to the + // empty database's block files before this scan, and when genesis + // does not fit in the file below a gap it rolls over into the + // missing file's number, so the scan reads past the hole. let mut orphaned = 0usize; let mut first_orphan_file: Option = None; for (hash, record) in &header_by_hash { @@ -10185,6 +10198,19 @@ pub(crate) fn network_magic(network: Network) -> [u8; 4] { } } +/// Where `genesis` sits when it is the first record of `blk00000.dat`, as a +/// fresh satd or Core datadir writes it: the record's payload must be the +/// genesis block byte for byte. `None` when the file is missing, unreadable, +/// or begins with anything else. +fn genesis_record_on_disk(flat_files: &mut FlatFileManager, genesis: &Block) -> Option { + let pos = FlatFilePos { + file_number: 0, + data_pos: 0, + }; + let payload = flat_files.read_block(&pos).ok()?; + (payload == serialize(genesis)).then_some(pos) +} + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -20791,6 +20817,116 @@ pub(crate) mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// Every `blk*.dat` under `blocks`, by name, with its bytes. + fn blk_file_bytes(blocks: &std::path::Path) -> Vec<(String, Vec)> { + let mut files: Vec<(String, Vec)> = std::fs::read_dir(blocks) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.starts_with("blk") && n.ends_with(".dat")) + .map(|n| { + let bytes = std::fs::read(blocks.join(&n)).unwrap(); + (n, bytes) + }) + .collect(); + files.sort(); + files + } + + /// #931: a full `-reindex` opens an empty store over block files that + /// already begin with genesis. Genesis is indexed where it is, so the + /// reindex leaves the block files, which may be Core's, as it found them. + /// + /// Perturbation: drop the `genesis_record_on_disk` branch from + /// `ChainState::new` and the last file grows by a second genesis record. + #[test] + fn a_full_reindex_leaves_the_block_files_as_it_found_them() { + let (cs, dir) = make_chain_state(); + let hashes = build_chain(&cs, 6, 1_702_100_000); + cs.flat_files.lock().sync_all().unwrap(); + let before = blk_file_bytes(&dir.join("blocks")); + + let re = reindexing_chain_state_over(&dir); + let genesis = bitcoin::constants::genesis_block(Network::Regtest).block_hash(); + let entry = re.store.get_block_index(&genesis).expect("genesis is indexed"); + assert_eq!((entry.file_number, entry.data_pos), (0, 0)); + let end = re.reindex_from_flat_files(None, None, None).expect("reindex"); + + assert_eq!(end, FlatFileReindexEnd::Finished); + assert_eq!(re.tip_hash(), hashes[5]); + assert!( + blk_file_bytes(&dir.join("blocks")) == before, + "the reindex wrote to the block files" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// #931: genesis is reused only where `blk00000.dat` begins with it, read + /// through the blocks directory's `xor.dat` key as Core 28+ writes one. A + /// first record that is anything else gets genesis appended, as a new + /// datadir does. + #[test] + fn genesis_is_reused_only_where_blk00000_begins_with_it() { + let genesis = bitcoin::constants::genesis_block(Network::Regtest); + let magic = network_magic(Network::Regtest); + let open = |blocks: &std::path::Path| { + ChainState::new( + Box::new(InMemoryStore::new()), + FlatFileManager::new(blocks).unwrap(), + Network::Regtest, + Box::new(NoopVerifier), + AssumeValid::Disabled, + 450, + 4, + Default::default(), + Default::default(), + Default::default(), + ) + .unwrap() + }; + for key in [[0u8; 8], [0x8f, 0x1a, 0x00, 0xc4, 0x5e, 0x21, 0xd0, 0x77]] { + for genesis_first in [true, false] { + let dir = tempfile::TempDir::new().unwrap(); + let blocks = dir.path().join("blocks"); + std::fs::create_dir_all(&blocks).unwrap(); + std::fs::write(blocks.join("xor.dat"), key).unwrap(); + { + let mut files = FlatFileManager::new(&blocks).unwrap(); + let first = if genesis_first { + serialize(&genesis) + } else { + b"a record that is not genesis".to_vec() + }; + files.write_block(&first, magic, 0).unwrap(); + files.sync_all().unwrap(); + } + let before = blk_file_bytes(&blocks); + + let cs = open(&blocks); + let entry = cs.store.get_block_index(&genesis.block_hash()).unwrap(); + let at = (entry.file_number, entry.data_pos); + let after = blk_file_bytes(&blocks); + let stored = cs.flat_files.lock().read_block(&FlatFilePos { + file_number: entry.file_number, + data_pos: entry.data_pos, + }); + + assert_eq!(stored.unwrap(), serialize(&genesis), "key {key:?}"); + if genesis_first { + assert_eq!(at, (0, 0), "key {key:?}"); + assert!(after == before, "key {key:?}: genesis was written again"); + } else { + assert_ne!(at, (0, 0), "key {key:?}"); + assert_eq!( + after[0].1.len(), + before[0].1.len() + 8 + serialize(&genesis).len(), + "key {key:?}: genesis is appended after the other record" + ); + } + } + } + } + /// #906: a fresh reindex refuses a chainstate that is not empty. Its /// connect starts at block 1, so over a stored chain it would fail there, /// or, over a stored chain that differs, build on it. From 31a56d616b85685fd896ae66bb1594e7f9b6a467 Mon Sep 17 00:00:00 2001 From: Benjamen Keroack Date: Wed, 7 Oct 2026 15:57:41 -0600 Subject: [PATCH 2/2] StartOS package: run satd with RUST_LOG=info (#930) StartOS starts every daemon with RUST_LOG=warn,start_core=debug in its environment, and satd takes its log filter from RUST_LOG over its own default of info. Under the package, the service log carried warnings only: no sync progress, no reindex phases, no shutdown steps. Found while testing the 0.6.1 reindex and stop fixes in a StartOS VM, where the process environment showed the inherited value. The satd daemon now sets RUST_LOG=info. A test checks the satd daemon's exec carries it, and that satd still reads RUST_LOG. Co-authored-by: Claude Opus 5.5 (1M context) (cherry picked from commit 33902a4639a90ed7c7de0c07f9476b1e86f79719) --- contrib/packaging/startos/startos/main.ts | 5 +++ .../packaging/startos/test/logging.test.ts | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 contrib/packaging/startos/test/logging.test.ts diff --git a/contrib/packaging/startos/startos/main.ts b/contrib/packaging/startos/startos/main.ts index 100e65908..1f768171e 100644 --- a/contrib/packaging/startos/startos/main.ts +++ b/contrib/packaging/startos/startos/main.ts @@ -151,6 +151,11 @@ export const main = sdk.setupMain(async ({ effects }) => { // `--chain=` because there are bare flags for the test networks but // none for mainnet. command: ['satd', `--datadir=${rootDir}`, `--chain=${network}`], + // StartOS runs every daemon with its own RUST_LOG + // (`warn,start_core=debug`), and satd honors RUST_LOG over its + // default of `info`. Without this the service log shows warnings + // only: no sync progress, no reindex phases, no shutdown steps. + env: { RUST_LOG: 'info' }, user: 'satd', // A node writing out its chainstate should not be killed mid-flush. sigtermTimeout: 600_000, diff --git a/contrib/packaging/startos/test/logging.test.ts b/contrib/packaging/startos/test/logging.test.ts new file mode 100644 index 000000000..ed0cb3938 --- /dev/null +++ b/contrib/packaging/startos/test/logging.test.ts @@ -0,0 +1,39 @@ +import { match, ok } from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { upstream } from './upstream.ts' + +const read = (p: string) => + readFileSync(new URL(p, import.meta.url), 'utf8') + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/(^|[^:])\/\/.*$/gm, '$1') + +/** + * StartOS starts every daemon with RUST_LOG=warn,start_core=debug in its + * environment, and satd reads RUST_LOG in preference to its own default of + * info. A package that passes no RUST_LOG gets a service log of warnings + * only: on a StartOS server the log said nothing of sync progress, a + * reindex's phases or the steps of a shutdown. + */ +test('satd runs with RUST_LOG=info, not the RUST_LOG StartOS hands it', () => { + const main = read('../startos/main.ts') + const start = main.indexOf(".addDaemon('satd',") + ok(start >= 0, "main.ts has no addDaemon('satd', …)") + // Up to the next step of the chain (.addHealthCheck, .addOneshot, …). + const rest = main.slice(start + 1) + const next = rest.search(/\.add[A-Z]\w*\(/) + const daemon = next < 0 ? rest : rest.slice(0, next) + match(daemon, /env:\s*\{[^}]*RUST_LOG:\s*'info'/) +}) + +const config = upstream('satd/src/config.rs') +test( + 'satd takes its log filter from RUST_LOG', + { skip: config === null && 'satd source not present' }, + () => { + ok( + config!.includes('std::env::var("RUST_LOG")'), + 'satd no longer reads RUST_LOG; the env line in main.ts may be moot', + ) + }, +)