From c058b8e9f5a3781b61d0965e737f58c79d2250f5 Mon Sep 17 00:00:00 2001 From: fireice-uk Date: Sat, 16 Mar 2024 13:55:30 +0000 Subject: [PATCH 01/56] Basic hash-of-hashes mechanism --- src/CryptoNoteCore/BlockIndex.cpp | 6 ++++++ src/CryptoNoteCore/BlockIndex.h | 1 + src/CryptoNoteCore/Blockchain.cpp | 7 ++++++- src/CryptoNoteCore/Blockchain.h | 3 ++- src/CryptoNoteCore/Core.cpp | 4 ++++ src/CryptoNoteCore/Core.h | 1 + src/Daemon/DaemonCommandsHandler.cpp | 20 ++++++++++++++++++++ src/Daemon/DaemonCommandsHandler.h | 3 ++- 8 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/CryptoNoteCore/BlockIndex.cpp b/src/CryptoNoteCore/BlockIndex.cpp index 243170f26..48a869336 100644 --- a/src/CryptoNoteCore/BlockIndex.cpp +++ b/src/CryptoNoteCore/BlockIndex.cpp @@ -34,6 +34,12 @@ namespace cn { return result; } + crypto::Hash BlockIndex::getHashOfIds(uint32_t startBlockIndex, uint32_t maxCount) const { + /* This can be made more efficient by hashing in-place instead of copying all hashes, but it needs three function hash code */ + std::vector block_ids = getBlockIds(startBlockIndex, maxCount); + return crypto::cn_fast_hash(block_ids.data(), block_ids.size() * sizeof(crypto::Hash)); + } + bool BlockIndex::findSupplement(const std::vector& ids, uint32_t& offset) const { for (const auto& id : ids) { if (getBlockHeight(id, offset)) { diff --git a/src/CryptoNoteCore/BlockIndex.h b/src/CryptoNoteCore/BlockIndex.h index c122c315e..d64b1a9ef 100644 --- a/src/CryptoNoteCore/BlockIndex.h +++ b/src/CryptoNoteCore/BlockIndex.h @@ -58,6 +58,7 @@ namespace cn crypto::Hash getBlockId(uint32_t height) const; std::vector getBlockIds(uint32_t startBlockIndex, uint32_t maxCount) const; + crypto::Hash getHashOfIds(uint32_t startBlockIndex, uint32_t maxCount) const; bool findSupplement(const std::vector& ids, uint32_t& offset) const; std::vector buildSparseChain(const crypto::Hash& startBlockId) const; crypto::Hash getTailId() const; diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index e97e5a1b6..c208715fc 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -3287,4 +3287,9 @@ namespace cn return m_checkpoints.is_in_checkpoint_zone(height); } -} // namespace cn \ No newline at end of file + crypto::Hash Blockchain::getCheckpointHash(uint32_t height) const + { + std::lock_guard lk(m_blockchain_lock); + return m_blockIndex.getHashOfIds(0, height); + } +} // namespace cn diff --git a/src/CryptoNoteCore/Blockchain.h b/src/CryptoNoteCore/Blockchain.h index e909bf146..7235aeae7 100644 --- a/src/CryptoNoteCore/Blockchain.h +++ b/src/CryptoNoteCore/Blockchain.h @@ -125,6 +125,7 @@ namespace cn uint64_t coinsEmittedAtHeight(uint64_t height); uint64_t difficultyAtHeight(uint64_t height); bool isInCheckpointZone(const uint32_t height) const; + crypto::Hash getCheckpointHash(uint32_t height) const; template bool scanOutputKeysForIndexes(const KeyInput &tx_in_to_key, visitor_t &vis, uint32_t *pmax_related_block_height = nullptr); @@ -482,4 +483,4 @@ namespace cn return true; } }; -} // namespace cn \ No newline at end of file +} // namespace cn diff --git a/src/CryptoNoteCore/Core.cpp b/src/CryptoNoteCore/Core.cpp index 99c3eb6a8..49ea45a8c 100644 --- a/src/CryptoNoteCore/Core.cpp +++ b/src/CryptoNoteCore/Core.cpp @@ -106,6 +106,10 @@ void core::get_blockchain_top(uint32_t& height, crypto::Hash& top_id) { top_id = m_blockchain.getTailId(height); } +crypto::Hash core::checkpoint_hash(uint32_t height) { + return m_blockchain.getCheckpointHash(height); +} + bool core::rollback_chain_to(uint32_t height) { return m_blockchain.rollbackBlockchainTo(height); } diff --git a/src/CryptoNoteCore/Core.h b/src/CryptoNoteCore/Core.h index 0be9257ff..4935f23f5 100644 --- a/src/CryptoNoteCore/Core.h +++ b/src/CryptoNoteCore/Core.h @@ -94,6 +94,7 @@ namespace cn { bool get_blocks(uint32_t start_offset, uint32_t count, std::list& blocks, std::list& txs); bool get_blocks(uint32_t start_offset, uint32_t count, std::list& blocks); bool rollback_chain_to(uint32_t height); + crypto::Hash checkpoint_hash(uint32_t height); template bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) { diff --git a/src/Daemon/DaemonCommandsHandler.cpp b/src/Daemon/DaemonCommandsHandler.cpp index ccb13ac84..61b966f7e 100644 --- a/src/Daemon/DaemonCommandsHandler.cpp +++ b/src/Daemon/DaemonCommandsHandler.cpp @@ -34,6 +34,7 @@ DaemonCommandsHandler::DaemonCommandsHandler(cn::core &core, cn::NodeServer &srv m_consoleHandler.setHandler("save", boost::bind(&DaemonCommandsHandler::save, this, boost::arg<1>()), "Save the Blockchain data safely"); m_consoleHandler.setHandler("print_pl", boost::bind(&DaemonCommandsHandler::print_pl, this, boost::arg<1>()), "Print peer list"); m_consoleHandler.setHandler("rollback_chain", boost::bind(&DaemonCommandsHandler::rollback_chain, this, boost::arg<1>()), "Rollback chain to specific height, rollback_chain "); + m_consoleHandler.setHandler("checkpoint_hash", boost::bind(&DaemonCommandsHandler::checkpoint_hash, this, boost::arg<1>()), "Calculate checkpoint hash up to "); m_consoleHandler.setHandler("print_cn", boost::bind(&DaemonCommandsHandler::print_cn, this, boost::arg<1>()), "Print connections"); m_consoleHandler.setHandler("print_bci", boost::bind(&DaemonCommandsHandler::print_bci, this, boost::arg<1>()), "Print blockchain current height"); m_consoleHandler.setHandler("print_bc", boost::bind(&DaemonCommandsHandler::print_bc, this, boost::arg<1>()), "Print blockchain info in a given blocks range, print_bc []"); @@ -309,6 +310,25 @@ bool DaemonCommandsHandler::rollback_chain(const std::vector &args) return true; } +bool DaemonCommandsHandler::checkpoint_hash(const std::vector &args) +{ + if (args.empty()) + { + logger(logging::ERROR) << "Usage: \"checkpoint_hash \""; + return true; + } + logger(logging::DEBUGGING) << "Attempting: checkpoint_hash"; + + const std::string &arg = args.front(); + uint32_t height = boost::lexical_cast(arg); + crypto::Hash hash = m_core.checkpoint_hash(height); + + logger(logging::INFO) << "Checkpoint hash: " << hash << " for " << height; + + logger(logging::DEBUGGING) << "Finished: checkpoint_hash"; + return true; +} + bool DaemonCommandsHandler::rollbackchainto(uint32_t height) { logger(logging::DEBUGGING) << "Attempting: rollbackchainto"; diff --git a/src/Daemon/DaemonCommandsHandler.h b/src/Daemon/DaemonCommandsHandler.h index f7bbc7088..7ad8c7dcc 100644 --- a/src/Daemon/DaemonCommandsHandler.h +++ b/src/Daemon/DaemonCommandsHandler.h @@ -52,7 +52,8 @@ class DaemonCommandsHandler bool show_hr(const std::vector& args); bool hide_hr(const std::vector& args); bool rollbackchainto(uint32_t height); - bool rollback_chain(const std::vector& args); + bool rollback_chain(const std::vector& args); + bool checkpoint_hash(const std::vector &args); bool print_cn(const std::vector& args); bool print_bc(const std::vector& args); bool print_bci(const std::vector& args); From 53b65b050f94a7c2181c173e968560913937883a Mon Sep 17 00:00:00 2001 From: fireice-uk Date: Fri, 22 Mar 2024 11:34:48 +0000 Subject: [PATCH 02/56] minor lint changes --- src/CryptoNoteCore/Core.cpp | 18 ++++++++++-------- src/CryptoNoteCore/Core.h | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/CryptoNoteCore/Core.cpp b/src/CryptoNoteCore/Core.cpp index 49ea45a8c..b9b83c3c6 100644 --- a/src/CryptoNoteCore/Core.cpp +++ b/src/CryptoNoteCore/Core.cpp @@ -60,20 +60,22 @@ class BlockWithTransactions : public IBlock { friend class core; }; -core::core(const Currency ¤cy, i_cryptonote_protocol *pprotocol, logging::ILogger &logger, bool blockchainIndexesEnabled, bool blockchainAutosaveEnabled) : m_currency(currency), - logger(logger, "core"), - m_mempool(currency, m_blockchain, m_timeProvider, logger), - m_blockchain(currency, m_mempool, logger, blockchainIndexesEnabled, blockchainAutosaveEnabled), - m_miner(new Miner(currency, *this, logger)), - m_starter_message_showed(false) +core::core(const Currency ¤cy, i_cryptonote_protocol *pprotocol, logging::ILogger &logger, bool blockchainIndexesEnabled, bool blockchainAutosaveEnabled) : + m_currency(currency), + logger(logger, "core"), + m_mempool(currency, m_blockchain, m_timeProvider, logger), + m_blockchain(currency, m_mempool, logger, blockchainIndexesEnabled, blockchainAutosaveEnabled), + m_miner(new Miner(currency, *this, logger)), + m_starter_message_showed(false) { set_cryptonote_protocol(pprotocol); m_blockchain.addObserver(this); m_mempool.addObserver(this); } - //----------------------------------------------------------------------------------------------- - core::~core() { + +//----------------------------------------------------------------------------------------------- +core::~core() { m_blockchain.removeObserver(this); } diff --git a/src/CryptoNoteCore/Core.h b/src/CryptoNoteCore/Core.h index 4935f23f5..11164284e 100644 --- a/src/CryptoNoteCore/Core.h +++ b/src/CryptoNoteCore/Core.h @@ -161,7 +161,7 @@ namespace cn { bool add_new_tx(const Transaction &tx, const crypto::Hash &tx_hash, size_t blob_size, tx_verification_context &tvc, bool keeped_by_block, uint32_t height); bool load_state_data(); bool parse_tx_from_blob(Transaction &tx, crypto::Hash &tx_hash, crypto::Hash &tx_prefix_hash, const BinaryArray &blob); - bool handle_incoming_block(const Block &b, block_verification_context &bvc, bool control_miner, bool relay_block); + bool handle_incoming_block(const Block &b, block_verification_context &bvc, bool control_miner, bool relay_block) override; bool check_tx_syntax(const Transaction &tx); //check correct values, amounts and all lightweight checks not related with database From 4b96cd6776603269e84ab4b04abcc09f4f85b9da Mon Sep 17 00:00:00 2001 From: fireice-uk Date: Thu, 28 Mar 2024 12:03:11 +0000 Subject: [PATCH 03/56] wip --- src/CryptoNoteConfig.h | 175 ++------------- src/CryptoNoteCore/Blockchain.cpp | 93 +++++--- src/CryptoNoteCore/Blockchain.h | 10 +- src/CryptoNoteCore/CheckpointList.h | 130 +++++++++++ src/CryptoNoteCore/Checkpoints.cpp | 203 ----------------- src/CryptoNoteCore/Checkpoints.h | 36 --- src/CryptoNoteCore/CheckpointsList.cpp | 209 ++++++++++++++++++ src/CryptoNoteCore/Core.cpp | 4 - src/CryptoNoteCore/Core.h | 9 +- src/CryptoNoteCore/Currency.cpp | 1 + src/CryptoNoteCore/Currency.h | 9 +- src/CryptoNoteCore/ICore.h | 3 + .../CryptoNoteProtocolDefinitions.h | 44 ++++ .../CryptoNoteProtocolHandler.cpp | 77 ++++++- .../CryptoNoteProtocolHandler.h | 6 +- src/Daemon/Daemon.cpp | 8 +- src/P2p/ConnectionContext.h | 2 + 17 files changed, 580 insertions(+), 439 deletions(-) create mode 100644 src/CryptoNoteCore/CheckpointList.h delete mode 100644 src/CryptoNoteCore/Checkpoints.cpp delete mode 100644 src/CryptoNoteCore/Checkpoints.h create mode 100644 src/CryptoNoteCore/CheckpointsList.cpp diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index 94ee5c69a..f596f03cd 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -141,6 +141,7 @@ namespace cn const char P2P_NET_DATA_FILENAME[] = "p2pstate.bin"; const char CRYPTONOTE_BLOCKCHAIN_INDICES_FILENAME[] = "blockchainindices.dat"; const char MINER_CONFIG_FILE_NAME[] = "miner_conf.json"; + const char CRYPTONOTE_CHECKPOINT_FILENAME[] = "checkpoint.dat"; } // namespace parameters @@ -185,12 +186,13 @@ namespace cn and the minimum version for communication between nodes */ const uint8_t P2P_VERSION_1 = 1; const uint8_t P2P_VERSION_2 = 2; - const uint8_t P2P_CURRENT_VERSION = 1; + const uint8_t P2P_CURRENT_VERSION = 2; const uint8_t P2P_MINIMUM_VERSION = 1; const uint8_t P2P_UPGRADE_WINDOW = 2; // This defines the minimum P2P version required for lite blocks propogation - const uint8_t P2P_LITE_BLOCKS_PROPOGATION_VERSION = 3; + const uint8_t P2P_LITE_BLOCKS_PROPOGATION_VERSION = 2; + const uint8_t P2P_CHECKPOINT_LIST_VERSION = 2; const size_t P2P_LOCAL_WHITE_PEERLIST_LIMIT = 1000; const size_t P2P_LOCAL_GRAY_PEERLIST_LIMIT = 5000; @@ -206,6 +208,7 @@ namespace cn const uint32_t P2P_DEFAULT_PING_CONNECTION_TIMEOUT = 2000; // 2 seconds const uint64_t P2P_DEFAULT_INVOKE_TIMEOUT = 60 * 2 * 1000; // 2 minutes const size_t P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT = 5000; // 5 seconds + const size_t P2P_CHECKPOINT_LIST_RE_REQUEST = 300; // 5 minutes const char P2P_STAT_TRUSTED_PUB_KEY[] = "f7061e9a5f0d30549afde49c9bfbaa52ac60afdc46304642b460a9ea34bf7a4e"; // Seed Nodes @@ -230,160 +233,28 @@ namespace cn __attribute__((unused)) #endif + const char DNS_CHECKPOINT_DOMAIN[] = "checkpoints.conceal.id"; + const char TESTNET_DNS_CHECKPOINT_DOMAIN[] = "testpoints.conceal.gq"; + // Blockchain Checkpoints: // {, ""}, const std::initializer_list CHECKPOINTS = { - {0, "b9dc432e56e37b52771970ce014dd23fda517cfd4fc5a9b296f1954b7d4505de"}, - {10000, "55cf271a5c97785fb35fea7ed177cb75f47c18688bd86fc01ae66508878029d6"}, - {20000, "52533de7f1596154c6954530ae8331fe4f92e92d476f097c6d7d20ebab1c2748"}, - {30000, "50b5d84ac0b8abfe25669aac8514505c4c5f7ffd8e2bba0b52ab64f600d90796"}, - {40000, "ae2ed29163a57396f11c743400e55fba3f6b8e6bb6473f421c48ff8c87447ad0"}, - {50000, "8ad7969ca5d3cf48f784d33b60d1ea00bfb35b632447584e5181b194f3bb9cd6"}, - {60000, "22b1a161de2318b1a83ae0e3d1d04a2c420accccadd861aa8ad6365ec630ce04"}, - {70000, "4ef8a3c59b04ad8ae335fee0b5df0c1b114dda57d13232741d82c4984bf22bed"}, - {80000, "a60bd6b446c5b09997b5b70f31c56f35358657a673dcf56213a163fb6516750d"}, - {90000, "9985f631d4b2c15388e8c3797a1384b4610b13ff3852bc6d8f125ea4e13fdd22"}, - {100000, "1ccef60fb31646fc1745ccb42167f2e2efcf953a83b99ff4b6a39c99eb37d0e5"}, - {110000, "1b80bf8355ea023de7ed3367881ef111dfdb3aaeb25db3a0d6cad4c3cc0bb4bd"}, - {120000, "f621bd615716b75716eb89714d60481077a16b1df4046bf829f6d09b1c8e58a6"}, - {130000, "deb2514d03e2faf1c63b55f707b1524665ca4bd71cace3f4e8f0de58f32ecc41"}, - {140000, "c439524c13187bb6008acb1e9999317aa44e8d1cd75c96faae78831f6b961bac"}, - {150000, "90cc70379ea81d47df998e8b9928ba9191968035ae79ec1cb429c64a55497e03"}, - {160000, "4176bdff06416934d7766a6c2f6279d048cfdc516019a0580ea19c1d003038cc"}, - {170000, "50e3af756e96115011c8e4d138852e1f4835da805ca5ccd826f81593a53f4bd3"}, - {180000, "e1672173a2794245830a742d1df38b5fe5006fe6f00707e1b776bf29316ab18b"}, - {190000, "763eaa3c049ef46479144924b41cc9cb37346da88b0a3ae32a10e026c6f7984c"}, - {200000, "2ef304bec067c3a94f04440a593a13903a1487890493d15f74ec79c0ae585109"}, - {210000, "90dd7aca026ec5f9fdfd2fa9cd0c114c1c6c6bfc0536fb6490c804ac7ef72425"}, - {220000, "8de5278fc6703933e32e062b14496b0e1562c941e7e3c5b93147a3b39491fac5"}, - {230000, "f8ed2680d912a7f3aeb452d4eb8023f93f6387ff4c6927615691f66701d05d32"}, - {240000, "4445874d16b3dd8d5b0f9dee287e47219022c2b214c459e03be2bb71e4a12e3d"}, - {250000, "c579d2ad4f95a6c34180a89b32aa9fbe6ab2ecba9f3714ddde90fd5d9f85f6e9"}, - {260000, "ce63d00de7546f1dee417b2391692b367dc5c2cfe19ea43c98cf932d3838c5ec"}, - {270000, "f16000fefb54ad1f0f927f634c5b6f44fcfa201adc5ee093850301bd773c18fa"}, - {280000, "aba16466e085b2c7a792ba449f025bd1e37d6a1d44fa957a1ad4df78f41f6478"}, - {290000, "9fd5f13ac51df7ce2b8d78c45fbb864b231d6275bf7495118b4cc415301e6fe1"}, - {290665, "4e0082f3e66b0fe4176a850ff9560f1d8d2f2e11dc3a2045904209d11478f779"}, - {290674, "ac89a1f4c20674a8d735681b1ded3a1242252bb23341bc9b79bc06b310b490f4"}, - {290675, "6782c5e7436f77f4466253d6a70466cc6bfc66c6c51b675864c4543250c09e8b"}, - {290676, "0b25026f8c7fb194776c081f2bb32874b82f4298bd0d71c2d0a986117b97fa1e"}, - {290720, "36572a88fbed4654f4291f6d7a35a732b81f61e87ec27ce58f38047981b84e09"}, - {300000, "2a984212cc42ef62cd2229b624e05aa72926f0e89006e976c88b52d99ea14225"}, - {305000, "46104ab66387ab6ca6a3889e81c7b9810e27f547a8684659aeb62c438a3b6cf0"}, - {310000, "4a896f5de4f782c59f1f4691505aba0df87a20f2e06499b59496b8d7ffb025fe"}, - {320000, "c68d15c181bdfc6c5b7fe5c46c6432a03b95d640caa425a5cb3aa675c1d8f8fd"}, - {330000, "af9e972f98bed57579a6691c3d21443d3cbff35005e984044bc99cee82d93922"}, - {340000, "6fce13dd473f3673cd08b28171902e281d7fdbbd8b8ba34e0019ae18f597d22f"}, - {350000, "f08aad1562ceee3a6c8147846bb3e5dd15b3168007f588ab68bd8ee816eb386d"}, - {360000, "cd910715be7dccc155ad3e8a6311f1bbcfaffe3ee25186c454ed27ee61faa977"}, - {370000, "6c4a86be9a1f697cadc38d21718803c43f49bf60c71ae253293e29ebac6efe31"}, - {380000, "620709892437c28deb72a56e6a91960f481aa682d8dd8652f792fb33e6683ef5"}, - {390000, "d2ff4c39b4aed7ef08a99a00b9823bed44581e866180ae3daa8b8e990b57ec63"}, - {400000, "9b7302daf5e5933b9a3e75a12651eaad83bea7d0058191cf65eb20985fe281c5"}, - {410000, "4f343219e57f78c1063f4b4c5be6cb5a10599d64d36e9f686f7046469a6c7e73"}, - {420000, "56b2fec8f7a55c9e2960d7224999c2e8c83a77f051931ba1673e071e7bcd6851"}, - {430000, "6d6e24f6c518c9cc24a05967fd1bbb3aeffb670fd7329d0a24053662a2305d9e"}, - {440000, "6a0138801d48150985045bc671c752f8209d084adad3624a57edd22f9edbef78"}, - {460000, "dec1da5df01c3cdf5d25a577816c93de58dfb6dd6b073619c5cbd50aedefceb7"}, - {470000, "1d07fd8995e17429143202da00138f0bfcbdd20aa5ddbba18ac762bc473ffd77"}, - {480000, "c896df9146e8f09f6205496dfa1e28037c8223f531546d2d64119068a6d1db1d"}, - {490000, "faa86e0b546f7655e829dcd8e967a52d9fa933c832863a648df30cc0e8771fa8"}, - {500000, "df5b2b47960ecd7809f037de44c6817640283e13323a36fe3dd894f3b2b3c5e1"}, - {510000, "db784d782ac463fbfbf221b417166a80ca1451f8895a1e3027bd19de2952c9bc"}, - {520000, "70b9c6945d8156d97d5f337b22ec8a4f77fa8af3b89d63e3fe6b834a03f7a613"}, - {530000, "f1b6f4018201e9c498e2b441f8e20f6e562e5d45c69008fe74caa7baa0a16611"}, - {550000, "1b922d13de891cd9f7224bd1a3c879a1d7634505f5f562623d7a487d44211327"}, - {570000, "9efe8868099afd1f6b17de773da0f5baebf2ace666bf5e599188c64d27cd429f"}, - {580000, "39ecee8d292c4e0440467b28ead6ed96c480ac85bec4fdba1e4c14b49b08077e"}, - {590000, "d6201b072cfed013b0e1091517624ca72bdd1ef147143356a1f951dd3241dd88"}, - {600000, "9f87dd161e37e9dbbcd86a3fafe8e1dec8c54194251ca0c36c646173db12c115"}, - {610000, "9c95678a27c5bde2b53efdae5c20a5528f134c4ff75737dee3e3d63b4d79c7ba"}, - {620000, "e5de278b0ea676855873663a32a2d21bc6d98cffcb133e249c8219fb0fcdc3eb"}, - {630000, "762c8269af35d53408d806d453b8ca6f19fc9e83048bb8d985502344f1d5e08a"}, - {640000, "24e1ac8aff3e1e7850c06a377c68b2ea3afe53477b710b988b6b456383a50081"}, - {650000, "4587f3196487cdf12e701bebe30340669374e39b6e0ca7a3c32d6b522be44570"}, - {660000, "8d8338dab606e4010f1fa53bc0ef268c98f63bf727150184bfedbea37c40026d"}, - {670000, "26350d735576a40e4d4e628b57186f4c7f85b3bea6c15f28554706f4c78c3837"}, - {680000, "6774c21beb0f4e2383069da967654ce4d26743f313aa7c705f222c055fcf0e05"}, - {690000, "33e1ddd732edfb8e850cdca304ae398a2eb495fd2a6876ff759725788f5b1135"}, - {700000, "a6b8e9707cd5ac93931b3fcc6bb516d11e7cb840bf49c8d3712bdeba605557be"}, - {710000, "922f1ca029163e58a24d6573e7de6bf9bcecc16ae164ebfd0285c0eda57d4eec"}, - {750000, "0e22dabd4379040815f078525ed02ae95e26ae92bc9eb35628a5d588e176b900"}, - {760000, "ecc64815b44b4c0c67340ff7e0d9fefee2cfbfbed10d61260c49bbe98aeb6ea2"}, - {770000, "7c97512a8ffdefa3e97809779841e48ffa6b68bd8a5bf90fcb59c1547f5ad90f"}, - {780000, "7acd1d5c843245bf1cc12a966f388111e2258c029595d31539b9257bb1217e61"}, - {790000, "580580b3b628b68532d9b141cc5e3299ebdc6e421c58ed155916c418144704ff"}, - {800000, "bb0dc113b1bfde3f06bfa341ba7d6de6ea82cebc86b98fc3d122b695e0bfdebe"}, - {810000, "c8a8e81032d66b137a99087ddd6ca6289040cb336012d4248ff616e1e7abb5b3"}, - {820000, "98a8373323adca6f09048177c774783150bcf25b62e26c804fd33461fc1af09b"}, - {830000, "7007d4331703233f48a3f1a2e824734d02ec81e7da46b7e00a2c354edb8de357"}, - {840000, "eae669f44a964ce5501b971139ee04657b62dc2535e9305857a1c9e2e839790a"}, - {850000, "e354bf503e8273a90d5d338ce78966ee820b69490b3a3183f0765281429d2f77"}, - {860000, "11b35a3f0c78686d75991d5bf65e868e565ea4ba15e96c9b0f643a4d9983eedf"}, - {870000, "21fef5a9cbb6ebe8ba271bdb55ed4d3ccd0468ab31a77f634cabc893e2a9661e"}, - {880000, "a861367503fec46c12f8ae957438a389c4de7b3c267cfdb21a7be29337885e10"}, - {890000, "092b29ab3369d0227239f0604d57ab91a3e1794ca3abe0c75fd5e69acb611a66"}, - {900000, "4cb49bba6abe10464db1075ed7125172e639ec9ee03f08ddb4dbed318d9dbfb9"}, - {910000, "2b3064cc74a3780e55c6eba250ec1e6ad6deb7ea6188430c07ba6fb3b60b63f0"}, - {920000, "961be71463b51c41fa5fbd43213b43b4b66173c26c1cd29d55881aae29a8ce07"}, - {930000, "511d2784b65e9ff0da55358834b88319b653aa1b5bf5a0f76c25c0467c16c536"}, - {940000, "781dba46e6a2d8a7ba4129e32e7764c519e011d8967672f7873599eb5449f760"}, - {950000, "5404a8e358ddf55c018bff01f4c112fc5ea291ac4429524d8b5c496ede697246"}, - {960000, "5133cf16ece3cba43a199c11dc80ecd81a8dbda4ce0c517f78a3e400efa6a730"}, - {970000, "e9ae491f24acd824dbcd43178c3070878e0ec32b494fe293b153fd19b2e8428a"}, - {980000, "c99c74ae4e3ff43ba2c93cd8a6d3ed52598c7bda6b42fac18d9569e29c5753aa"}, - {990000, "0412793a1650aeba2cf9ab7c32bee26668e8e997a55e97d65c609e903f9e04ec"}, - {1000000, "6ad9d4ccc9666b31481079374e573c20ebdf2d63862da8fcc2c45d13093b93ba"}, - {1010000, "f341d678cfbd5d488bbc179bc54fc92587dad7fb29823facc95f3e26158a722d"}, - {1020000, "e3dae82d451358ac300e3960695784efb7d76833e620d75196cd0af9db0568bb"}, - {1030000, "4d1d4f3174e684c93cb3dc0e261776224b02f6c0bd2c7ea91b6f923b42e7b321"}, - {1040000, "c894d5f5a6637d7ec50c9a09722059aa5e878bf1eda7210a1f6c4e61eca770ef"}, - {1050000, "8a0f5df47ce13a082423743651f22209cef78f46b933ced7642d1f6b8d8d80be"}, - {1060000, "59d6ee913234a03eefa023a3a12487c5244973e17a9c9dcc61bd7cb3c5dcc426"}, - {1070000, "0895514f95977bdbb220550f00ec38962a6e672a125dad115f2408600fd9b593"}, - {1080000, "58a71c6f06c16e3337f5dd43a018d2768b2f6ca2d97a7bee9e81b2b2bae866c3"}, - {1090000, "2c268f5a834ac104b04991b8b131205468ed06cd6a5acdadffa05db82394f113"}, - {1100000, "8b87b5d9941cf9b46ceaff134cff5bd8a9d0326bb045e8abcaa1eb63fda739bd"}, - {1110000, "2215193bc9e56654777e40c4d62c7b89f48e8d14b62335a800034bd5bed12835"}, - {1120000, "35a3baaaf080bd2e8321a0344fc939fbeca147f6078bc3c807cc3eec23325f96"}, - {1130000, "ea86d90d85c8b56edd03365d8558acb84dc3f33764a32df426eaf20b7d5b9d71"}, - {1140000, "611dd25b2ec9cc5a630e5aee561921efc13c939f179507f7a08afceb13f5035f"}, - {1150000, "122a7bf817cddd406a016304b35991f6f2a5cdd122cdc1b5a54fdd4e012066ae"}, - {1160000, "85b4997808f2c21ba7f94baa97de19bf1dc809fcd40936bf1fcaaca191ae1466"}, - {1170000, "7675f0144db2ae5f6e0c378a25778850c9fd6facd285e89cdfc4169f19bb79d2"}, - {1180000, "4f9c52d49f270df62ddf2b45f7e616321b686d6204e3c67f8807129ff180cdfb"}, - {1190000, "8b6fd535bec46f3b28772e82e210fd1988b1b4704801b75712c6af17b9a09a2b"}, - {1200000, "9434fdddd2e7521fa92aea42f11a30d364ccc413074993b2aacb31987cca02da"}, - {1210000, "d424b084320d8c138dca36de8bb2dba40e142f9fe6785f50b39c80543a653c4e"}, - {1220000, "9dad391d2c819b482d9a9408db63e480726597f21d180f9a590f3c20f2c9fbc8"}, - {1230000, "3c0fda332a0a1d0d9329087430960fd6506f7d01cf14cd027567074c3c9b79ef"}, - {1240000, "00a5a0b11c0f20fd30e98881dcf7c2246a67aea8529c71400f3bae4658214ce2"}, - {1250000, "786f2af5d5ff526cfb0330baed655d2a7bafbe5bcafa10759b17d6cd9cf9ee1c"}, - {1260000, "0aded508302e4987d6a585990e2b1f27b81582dd0ccff3fc40ca0779daaa7da1"}, - {1270000, "0b8a495699b85eae4e115b84259b056d9084025a0e9b11b67cca1c47552f0998"}, - {1280000, "544b164dfc4e19221e23e427fe821c812dda2a41fe5eee1915065f99a3c707d2"}, - {1290000, "7844314fdd7e70ed38def90377339e249108e328bb94820f070c6aa2a787abf9"}, - {1300000, "68e1ce32210bc1cd41ea26e64382f39c5302bf251273cc6fd35d80a3c19df815"}, - {1310000, "64aab7bbc148131f11479e74bbbd74c67e6ee45312bd6e72f5b68d5d1d383e46"}, - {1320000, "1e3d88026216db2a09b7771f5c36b6a9cf49086d259160b2ad4530155265c394"}, - {1330000, "2c05b06c4412738a06961406cea1d3b605afd0380a29ff306db6f1b820ce802f"}, - {1340000, "271352a6b9d8cdef191aaf3395d5b38d7e7bd1b270766aceec915d0e5d6eb9c4"}, - {1350000, "2911699e718b064a4820a860bff5b421ee122707cd4cc7bacebfcb70ed4ea8a5"}, - {1360000, "2635c819e25ea4d25be05adf7b515c09b14ff0bffd600921ab488d8597e1e35e"}, - {1370000, "a8e01900ca2289971a6f926c9bbd420a86c9213c287b8c207c4ee5accff27f10"}, - {1380000, "d6014404b60a7fd16372aacefb78132d417d8e08b5a6876851ab4862b167c84a"}, - {1390000, "30c3eb1c67b1ff52e84c7637131f6e0a950464c633e6e69bb27be7488a690462"}, - {1400000, "9668035887ed9f819382db025f852e57aa02fa8980b4c9043f0dd535ea4a1085"}, - {1410000, "a01f353bfe5b907f9b0ffa7e4226caca5d8a67e2bf4b39c13c93b63a4a7d4379"}, - {1420000, "930ad5850a8aeeb8ee38b08006bfea2e93474d97bf666b97df9ab9fbe84a79be"}, - {1430000, "8d1b006f9fa277196f62a98a501ecf91a81731d199463c23931181b4a28b694a"}, - {1440000, "b54921b7c396e66e1a15492289e33993a166f49675739f437257fbb760ee1035"}, - {1450000, "f8a2b95f394f6cd188363a20f585dadbfa0db707dd5fa2699604eb7ccab313a6"}, - {1460000, "6bd96b76bd2d3bc7ca320c089ffe21e64565432401cb94736d60d8f70cfb42f3"}, - {1470000, "7e2d26dd0b258ff826d6ff1e36fe6606206fca535ba0ed40e944ddb165da9dc0"}, - {1480000, "f4463eed0890245dca0ccf9fb3a9f101e110f1dc53ce1286ef47d56938faf007"}, - {1490000, "bc76acf39ea16ce02588798b93034208881bab6c6dd9dfc545a6fab51dfce886"} + { 100000, "9b58762e759cd02cef493b310f95d73c36a907ea5c6ab3953b6b304651d3f291"}, + { 200000, "07a7d796f64309b84558b7fc44902dd65dbf1bfd4b727b9f48b9f358b9c7e4f5"}, + { 300000, "d723e8964fd416d1fbb5c8d616b0f7aa2f61cd7dbf1c7f6654daf751915f7967"}, + { 400000, "1b4fbfd19b8502af420b8a38a6ac610a6b9e30a164af4da742092bdf9887d086"}, + { 500000, "47cce8323f661b07048be52a2c6cca29f9f49aa5cc3282253f6e0a54dd3f1d56"}, + { 600000, "086035e107dd22b8be63a86c28f750124306914cd47f46d0c69208580a4d5f9e"}, + { 700000, "b499559416d7198a01bfe6c02d94f0e0c4ff785158ffa6f690e736317372acc3"}, + { 800000, "3c5207312c528b80df4392f5f6a99cb23457d018c08747b072e7ab86a83025b4"}, + { 900000, "4bb3c7c5b7bd24dac9440f3f5797348716b1b1a6335a0de0769f0ea0f409e447"}, + {1000000, "3148b677584d71f5e75b5c7431d525aa6b8d8a7d5e4d01ea4adadc75adcc64d5"}, + {1100000, "480f890918c2c550696109391b55ca465fdc04a4302b403a90ccf00c37a282d9"}, + {1200000, "f730a854601d55c774755a6d8f561ab2454dfb763340f23b7218c24ae14163f4"}, + {1300000, "d130491c88398a2812849d27fa09b884085d7f2feffe5d5da0f56cd8039469f5"}, + {1400000, "570682ba6a7614081071a9f111b7e737789e1c7d445635bcd1f830211a311a19"}, + {1500000, "43f173aba14a6b2d023c07796683789470395d7f15ebd24991617b4cc81c4f8c"} }; const std::initializer_list TESTNET_CHECKPOINTS = { diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index c208715fc..2095410d0 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -355,7 +355,6 @@ namespace cn m_blockchainIndexesEnabled(blockchainIndexesEnabled), m_blockchainAutosaveEnabled(blockchainAutosaveEnabled), logger(logger, "Blockchain") - { } @@ -461,7 +460,6 @@ namespace cn bool Blockchain::init(const std::string &config_folder, bool load_existing, bool testnet) { m_testnet = testnet; - m_checkpoints.set_testnet(testnet); std::lock_guard lk(m_blockchain_lock); if (!config_folder.empty() && !tools::create_directories_if_necessary(config_folder)) { @@ -471,6 +469,9 @@ namespace cn m_config_folder = config_folder; + m_checkpoints.init_targets(testnet, appendPath(config_folder, m_currency.checkpointFileName())); + m_checkpoints.load_checkpoints_from_file(); + if (!m_blocks.open(appendPath(config_folder, m_currency.blocksFileName()), appendPath(config_folder, m_currency.blockIndexesFileName()), 1024)) { return false; @@ -593,25 +594,44 @@ namespace cn bool Blockchain::checkCheckpoints(uint32_t &lastValidCheckpointHeight) { - std::vector checkpointHeights = m_checkpoints.getCheckpointHeights(); - for (const auto &checkpointHeight : checkpointHeights) + bool rv = true; + std::vector> checkpointHeights = m_checkpoints.get_checkpoint_targets(); + lastValidCheckpointHeight = 0; + + /* Hashes are in reverse order so if everything is ok we will only do one iteration */ + for (const auto &check : checkpointHeights) { - if (m_blocks.size() <= checkpointHeight) - { - return true; - } + uint32_t height = check.first; + if (m_blocks.size() <= height) + continue; - if (m_checkpoints.check_block(checkpointHeight, getBlockIdByHeight(checkpointHeight))) + crypto::Hash hv = m_blockIndex.getHashOfIds(0, height+1); + if (hv == check.second) { - lastValidCheckpointHeight = checkpointHeight; + lastValidCheckpointHeight = height; + break; } else { - return false; + logger(ERROR, BRIGHT_RED) << "Checkpoint failed for " << height << " got " << + hv << " expected " << check.second; + rv = false; } } - logger(INFO, BRIGHT_WHITE) << "Checkpoints passed"; - return true; + + if(rv) + { + logger(INFO, BRIGHT_WHITE) << "Checkpoints passed " << m_checkpoints.get_points_size() << " " << lastValidCheckpointHeight; + uint32_t n_valid_blocks = lastValidCheckpointHeight+1; + if(m_checkpoints.get_points_size() < n_valid_blocks) + m_checkpoints.set_checkpoint_list(m_blockIndex.getBlockIds(0, n_valid_blocks)); + } + else + { + logger(ERROR, BRIGHT_RED) << "Checkpoints failed, last valid height " << lastValidCheckpointHeight; + } + + return rv; } void Blockchain::rebuildCache() @@ -1403,6 +1423,29 @@ namespace cn return true; } + bool Blockchain::is_alternative_block_allowed(uint32_t blockchain_height, uint32_t block_height) const { + if (0 == block_height) + return false; + + uint32_t lowest_height = blockchain_height - cn::parameters::CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; + + if (blockchain_height < cn::parameters::CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW) + { + lowest_height = 0; + } + + if (block_height < lowest_height && !m_checkpoints.is_in_checkpoint_zone(block_height)) + { + logger(logging::DEBUGGING, logging::WHITE) + << "<< Checkpoints.cpp << " + << "Reorganization depth too deep : " << (blockchain_height - block_height) << ". Block Rejected"; + return false; + } + + uint32_t checkpoint_height = m_checkpoints.get_greatest_target_height(); + return checkpoint_height < block_height; + } + bool Blockchain::handle_alternative_block(const Block &b, const crypto::Hash &id, block_verification_context &bvc, bool sendNewAlternativeBlockMessage) { std::lock_guard lk(m_blockchain_lock); @@ -1416,10 +1459,8 @@ namespace cn return false; } - /* in the absence of a better solution, we fetch checkpoints from dns records */ - m_checkpoints.load_checkpoints_from_dns(); - if (!m_checkpoints.is_alternative_block_allowed(getCurrentBlockchainHeight(), block_height)) + if (!is_alternative_block_allowed(getCurrentBlockchainHeight(), block_height)) { logger(DEBUGGING) << "Block with id: " << id << std::endl << " can't be accepted for alternative chain, block height: " << block_height << std::endl @@ -1514,8 +1555,8 @@ namespace cn bei.bl = b; bei.height = alt_chain.size() ? it_prev->second.height + 1 : mainPrevHeight + 1; - bool is_a_checkpoint; - if (!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) + auto checkpoint_status = m_checkpoints.check_checkpoint(bei.height, id); + if ( checkpoint_status == CheckpointList::is_in_zone_failed ) { logger(ERROR, BRIGHT_RED) << "Checkpoint validaton failure"; @@ -1581,7 +1622,7 @@ namespace cn alt_chain.push_back(i_res.first->first); - if (is_a_checkpoint) + if ( checkpoint_status == CheckpointList::is_checkpointed ) { //do reorganize! logger(INFO, BRIGHT_GREEN) << "###### REORGANIZE on height: " << m_alternative_chains[alt_chain.front()].height << " of " << m_blocks.size() - 1 << ", checkpoint is found in alternative chain on height " << bei.height; @@ -2481,15 +2522,13 @@ namespace cn auto longhashTimeStart = std::chrono::steady_clock::now(); crypto::Hash proof_of_work = NULL_HASH; - if (m_checkpoints.is_in_checkpoint_zone(getCurrentBlockchainHeight())) + auto checkpoint_status = m_checkpoints.check_checkpoint(getCurrentBlockchainHeight(), blockHash); + if (checkpoint_status == CheckpointList::is_in_zone_failed ) { - if (!m_checkpoints.check_block(getCurrentBlockchainHeight(), blockHash)) - { - bvc.m_verification_failed = true; - return false; - } + bvc.m_verification_failed = true; + return false; } - else + else if(checkpoint_status == CheckpointList::is_out_of_zone ) { if (!m_currency.checkProofOfWork(m_cn_context, blockData, currentDifficulty, proof_of_work)) { @@ -3290,6 +3329,6 @@ namespace cn crypto::Hash Blockchain::getCheckpointHash(uint32_t height) const { std::lock_guard lk(m_blockchain_lock); - return m_blockIndex.getHashOfIds(0, height); + return m_blockIndex.getHashOfIds(0, height+1); } } // namespace cn diff --git a/src/CryptoNoteCore/Blockchain.h b/src/CryptoNoteCore/Blockchain.h index 7235aeae7..b87b67359 100644 --- a/src/CryptoNoteCore/Blockchain.h +++ b/src/CryptoNoteCore/Blockchain.h @@ -15,7 +15,7 @@ #include "Common/ObserverManager.h" #include "Common/Util.h" #include "CryptoNoteCore/BlockIndex.h" -#include "CryptoNoteCore/Checkpoints.h" +#include "CryptoNoteCore/CheckpointList.h" #include "CryptoNoteCore/Currency.h" #include "CryptoNoteCore/DepositIndex.h" #include "CryptoNoteCore/IBlockchainStorageObserver.h" @@ -68,7 +68,6 @@ namespace cn bool getLowerBound(uint64_t timestamp, uint64_t startOffset, uint32_t &height); std::vector getBlockIds(uint32_t startHeight, uint32_t maxCount); - void setCheckpoints(Checkpoints &&chk_pts) { m_checkpoints = std::move(chk_pts); } bool getBlocks(uint32_t start_offset, uint32_t count, std::list &blocks, std::list &txs); bool getBlocks(uint32_t start_offset, uint32_t count, std::list &blocks); bool getAlternativeBlocks(std::list &blocks); @@ -84,6 +83,10 @@ namespace cn bool haveTransaction(const crypto::Hash &id); bool haveTransactionKeyImagesAsSpent(const Transaction &tx); + CheckpointList& getCheckpointList() { + return m_checkpoints; + } + uint32_t getCurrentBlockchainHeight(); // TODO rename to getCurrentBlockchainSize crypto::Hash getTailId(); crypto::Hash getTailId(uint32_t &height); @@ -285,7 +288,7 @@ namespace cn outputs_container m_outputs; std::string m_config_folder; - Checkpoints m_checkpoints; + CheckpointList m_checkpoints; std::atomic m_is_in_checkpoint_zone; using Blocks = SwappedVector; @@ -320,6 +323,7 @@ namespace cn bool switch_to_alternative_blockchain(const std::list &alt_chain, bool discard_disconnected_chain); + bool is_alternative_block_allowed(uint32_t blockchain_height, uint32_t block_height) const; bool handle_alternative_block(const Block &b, const crypto::Hash &id, block_verification_context &bvc, bool sendNewAlternativeBlockMessage = true); difficulty_type get_next_difficulty_for_alternative_chain(const std::list &alt_chain, const BlockEntry &bei); void pushToDepositIndex(const BlockEntry &block, uint64_t interest); diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h new file mode 100644 index 000000000..efa0d02d7 --- /dev/null +++ b/src/CryptoNoteCore/CheckpointList.h @@ -0,0 +1,130 @@ +// Copyright (c) 2011-2017 The Cryptonote developers +// Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs +// Copyright (c) 2018-2023 Conceal Network & Conceal Devs +// +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once +#include +#include + +#include "CryptoNoteBasicImpl.h" +#include + +namespace cn +{ + class CheckpointList + { + public: + explicit CheckpointList(logging::ILogger& log) : logger(log, "checkpoint_list") {} + + void init_targets(bool is_testnet, const std::string& save_file); + + bool add_checkpoint_list(uint32_t start_height, std::vector& points); + bool set_checkpoint_list(std::vector&& points); + bool load_checkpoints_from_file(); + + uint32_t get_points_size() const + { + const std::lock_guard lock(m_points_lock); + return m_points.size(); + } + + uint32_t get_greatest_target_height() const + { + return m_targets.rbegin()->first - 1; + } + + bool is_ready() const + { + const std::lock_guard lock(m_points_lock); + return m_points.size()-1 >= get_greatest_target_height(); + } + + bool is_in_checkpoint_zone(uint32_t height) const + { + const std::lock_guard lock(m_points_lock); + return m_points.size() < height; + } + + enum check_rt + { + is_out_of_zone, + is_in_zone_failed, + is_checkpointed + }; + + check_rt check_checkpoint(uint32_t height, const crypto::Hash& hv) const + { + const std::lock_guard lock(m_points_lock); + + if(m_points.size() <= height) + return is_out_of_zone; + if(m_points[height] == hv) + return is_checkpointed; + else + return is_in_zone_failed; + } + + std::vector> get_checkpoint_targets() const + { + std::vector> rv; + rv.reserve(m_targets.size()); + for(auto it = m_targets.rbegin(); it != m_targets.rend(); ++it) + rv.emplace_back(it->first-1, it->second); + return rv; + } + + struct t_get_incomplete_checkpoint_target_rv { + crypto::Hash target_hash = NULL_HASH; + uint32_t start_height; + uint32_t end_height; + }; + + t_get_incomplete_checkpoint_target_rv get_incomplete_checkpoint_target() const + { + t_get_incomplete_checkpoint_target_rv rv; + size_t point_size = m_points.size(); + if(point_size >= m_targets.rbegin()->first) + return rv; + rv.start_height = 0; + for(const auto& p : m_targets) + { + if(point_size < p.first) + { + rv.end_height = p.first; + rv.target_hash = p.second; + return rv; + } + else + { + rv.start_height = p.first; + } + } + return rv; + } + + + private: + bool m_testnet; + logging::LoggerRef logger; + std::string m_save_file; + + mutable std::mutex m_points_lock; + std::map m_targets; /*NB uint32_t is size not height */ + std::unordered_set m_valid_point_sizes; + std::vector m_points; + + bool save_checkpoints(); + bool add_checkpoint_target(uint32_t height, const std::string &hash_str); + bool is_fsize_valid(uint32_t fsize) + { + if(fsize % sizeof(crypto::Hash) != 0) + return false; + fsize /= sizeof(crypto::Hash); + + return m_valid_point_sizes.find(fsize) != m_valid_point_sizes.end(); + } + }; +} diff --git a/src/CryptoNoteCore/Checkpoints.cpp b/src/CryptoNoteCore/Checkpoints.cpp deleted file mode 100644 index a75d6dce1..000000000 --- a/src/CryptoNoteCore/Checkpoints.cpp +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright (c) 2011-2017 The Cryptonote developers -// Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -// Copyright (c) 2018-2023 Conceal Network & Conceal Devs -// -// Copyright (c) 2016-2019, The Karbo developers - -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Checkpoints.h" -#include "../CryptoNoteConfig.h" -#include "Common/StringTools.h" -#include "Common/DnsTools.h" - -using namespace logging; - -namespace cn { -//--------------------------------------------------------------------------- -Checkpoints::Checkpoints(logging::ILogger &log) : logger(log, "checkpoints") {} -//--------------------------------------------------------------------------- -bool Checkpoints::add_checkpoint(uint32_t height, const std::string &hash_str) { - crypto::Hash h = NULL_HASH; - - if (!common::podFromHex(hash_str, h)) { - logger(ERROR) << "<< Checkpoints.cpp << " << "Incorrect hash in checkpoints"; - return false; - } - - if (!(0 == m_points.count(height))) { - logger(DEBUGGING) << "Checkpoint already exists for height " << height; - return false; - } - - m_points[height] = h; - - return true; -} -//--------------------------------------------------------------------------- -bool Checkpoints::is_in_checkpoint_zone(uint32_t height) const { - return !m_points.empty() && (height <= (--m_points.end())->first); -} -//--------------------------------------------------------------------------- -bool Checkpoints::check_block(uint32_t height, const crypto::Hash &h, bool &is_a_checkpoint) const { - auto it = m_points.find(height); - is_a_checkpoint = it != m_points.end(); - if (!is_a_checkpoint) - return true; - - if (it->second == h) { - return true; - } else { - logger(logging::ERROR) << "<< Checkpoints.cpp << " << "Checkpoint failed for height " << height - << ". Expected hash: " << it->second - << ", Fetched hash: " << h; - return false; - } -} -//--------------------------------------------------------------------------- -bool Checkpoints::check_block(uint32_t height, const crypto::Hash &h) const { - bool ignored; - return check_block(height, h, ignored); -} -//--------------------------------------------------------------------------- -bool Checkpoints::is_alternative_block_allowed(uint32_t blockchain_height, uint32_t block_height) const { - if (0 == block_height) - return false; - - uint32_t lowest_height = blockchain_height - cn::parameters::CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; - - if (blockchain_height < cn::parameters::CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW) - { - lowest_height = 0; - } - - if (block_height < lowest_height && !is_in_checkpoint_zone(block_height)) - { - logger(logging::DEBUGGING, logging::WHITE) - << "<< Checkpoints.cpp << " - << "Reorganization depth too deep : " << (blockchain_height - block_height) << ". Block Rejected"; - return false; - } - - auto it = m_points.upper_bound(blockchain_height); - if (it == m_points.begin()) - return true; - - --it; - uint32_t checkpoint_height = it->first; - return checkpoint_height < block_height; -} - -//--------------------------------------------------------------------------- - -std::vector Checkpoints::getCheckpointHeights() const { - std::vector checkpointHeights; - checkpointHeights.reserve(m_points.size()); - for (const auto& it : m_points) { - checkpointHeights.push_back(it.first); - } - - return checkpointHeights; -} - -bool Checkpoints::load_checkpoints_from_dns() -{ - std::string domain("checkpoints.conceal.id"); - if (m_testnet) - { - domain = "testpoints.conceal.gq"; - } - std::vectorrecords; - - logger(logging::DEBUGGING) << "<< Checkpoints.cpp << " << "Fetching DNS checkpoint records from " << domain; - - if (!common::fetch_dns_txt(domain, records)) { - logger(logging::DEBUGGING) << "<< Checkpoints.cpp << " << "Failed to lookup DNS checkpoint records from " << domain; - } - - for (const auto& record : records) { - uint32_t height; - crypto::Hash hash = NULL_HASH; - std::stringstream ss; - size_t del = record.find_first_of(':'); - std::string height_str = record.substr(0, del), hash_str = record.substr(del + 1, 64); - ss.str(height_str); - ss >> height; - char c; - if (del == std::string::npos) continue; - if ((ss.fail() || ss.get(c)) || !common::podFromHex(hash_str, hash)) { - logger(logging::INFO) << "<< Checkpoints.cpp << " << "Failed to parse DNS checkpoint record: " << record; - continue; - } - - if (!(0 == m_points.count(height))) { - logger(DEBUGGING) << "<< Checkpoints.cpp << " << "Checkpoint already exists for height: " << height << ". Ignoring DNS checkpoint."; - } else { - add_checkpoint(height, hash_str); - logger(DEBUGGING) << "<< Checkpoints.cpp << " << "Added DNS checkpoint: " << height_str << ":" << hash_str; - } - } - - return true; -} - -bool Checkpoints::load_checkpoints() -{ - if (m_testnet) - { - for (const auto &cp : cn::TESTNET_CHECKPOINTS) - { - add_checkpoint(cp.height, cp.blockId); - } - } - else - { - for (const auto &cp : cn::CHECKPOINTS) - { - add_checkpoint(cp.height, cp.blockId); - } - } - return true; -} - -bool Checkpoints::load_checkpoints_from_file(const std::string& fileName) { - std::ifstream file(fileName); - if (!file) { - logger(logging::ERROR, BRIGHT_RED) << "Could not load checkpoints file: " << fileName; - return false; - } - std::string indexString; - std::string hash; - uint32_t height; - while (std::getline(file, indexString, ','), std::getline(file, hash)) { - try { - height = std::stoi(indexString); - } catch (const std::invalid_argument &) { - logger(ERROR, BRIGHT_RED) << "Invalid checkpoint file format - " - << "could not parse height as a number"; - return false; - } - if (!add_checkpoint(height, hash)) { - return false; - } - } - logger(logging::INFO) << "Loaded " << m_points.size() << " checkpoints from " << fileName; - return true; -} - -void Checkpoints::set_testnet(bool testnet) { m_testnet = testnet; } - -} diff --git a/src/CryptoNoteCore/Checkpoints.h b/src/CryptoNoteCore/Checkpoints.h deleted file mode 100644 index 6de13aca1..000000000 --- a/src/CryptoNoteCore/Checkpoints.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2011-2017 The Cryptonote developers -// Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -// Copyright (c) 2018-2023 Conceal Network & Conceal Devs -// -// Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#pragma once -#include -#include "CryptoNoteBasicImpl.h" -#include - -namespace cn -{ - class Checkpoints - { - public: - explicit Checkpoints(logging::ILogger& log); - - bool add_checkpoint(uint32_t height, const std::string& hash_str); - bool is_in_checkpoint_zone(uint32_t height) const; - bool load_checkpoints_from_file(const std::string& fileName); - bool load_checkpoints_from_dns(); - bool load_checkpoints(); - bool check_block(uint32_t height, const crypto::Hash& h) const; - bool check_block(uint32_t height, const crypto::Hash& h, bool& is_a_checkpoint) const; - bool is_alternative_block_allowed(uint32_t blockchain_height, uint32_t block_height) const; - std::vector getCheckpointHeights() const; - void set_testnet(bool testnet); - - private: - bool m_testnet = false; - std::map m_points; - logging::LoggerRef logger; - }; -} diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp new file mode 100644 index 000000000..ca15b1ac9 --- /dev/null +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -0,0 +1,209 @@ +// Copyright (c) 2011-2017 The Cryptonote developers +// Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs +// Copyright (c) 2018-2023 Conceal Network & Conceal Devs +// +// Copyright (c) 2016-2019, The Karbo developers + +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CheckpointList.h" +#include "../CryptoNoteConfig.h" +#include "Common/StringTools.h" +#include "Common/DnsTools.h" +#include "crypto/hash.h" + +using namespace logging; + +namespace cn { + void CheckpointList::init_targets(bool is_testnet, const std::string& save_file) + { + m_testnet = is_testnet; + m_save_file = save_file; + + for (const auto &cp : m_testnet ? cn::TESTNET_CHECKPOINTS : cn::CHECKPOINTS) + { + add_checkpoint_target(cp.height, cp.blockId); + } + + const char* domain; + if (m_testnet) + domain = TESTNET_DNS_CHECKPOINT_DOMAIN; + else + domain = DNS_CHECKPOINT_DOMAIN; + + std::vectorrecords; + + logger(logging::DEBUGGING) << "<< CheckpointList.cpp << " << "Fetching DNS checkpoint records from " << domain; + + if (!common::fetch_dns_txt(domain, records)) { + logger(logging::DEBUGGING) << "<< CheckpointList.cpp << " << "Failed to lookup DNS checkpoint records from " << domain; + } + + for (const auto& record : records) { + uint32_t height; + crypto::Hash hash = NULL_HASH; + std::stringstream ss; + size_t del = record.find_first_of(':'); + std::string height_str = record.substr(0, del), hash_str = record.substr(del + 1, 64); + ss.str(height_str); + ss >> height; + char c; + if (del == std::string::npos) continue; + if ((ss.fail() || ss.get(c)) || !common::podFromHex(hash_str, hash)) { + logger(logging::INFO) << "<< CheckpointList.cpp << " << "Failed to parse DNS checkpoint record: " << record; + continue; + } + + if (!(0 == m_targets.count(height))) { + logger(DEBUGGING) << "<< CheckpointList.cpp << " << "Checkpoint already exists for height: " << height << ". Ignoring DNS checkpoint."; + } else { + add_checkpoint_target(height, hash_str); + logger(DEBUGGING) << "<< CheckpointList.cpp << " << "Added DNS checkpoint target: " << height_str << ":" << hash_str; + } + } + } + + bool CheckpointList::add_checkpoint_target(uint32_t height, const std::string &hash_str) { + crypto::Hash h = NULL_HASH; + + if (!common::podFromHex(hash_str, h)) { + logger(ERROR) << "<< Checkpoints.cpp << " << "Incorrect hash in checkpoints"; + return false; + } + + if (!(0 == m_targets.count(height))) { + logger(DEBUGGING) << "Checkpoint already exists for height " << height; + return false; + } + + height += 1; + m_targets[height] = h; + m_valid_point_sizes.insert(height); + + return true; + } + + bool CheckpointList::set_checkpoint_list(std::vector&& points) + { + const std::lock_guard lock(m_points_lock); + uint32_t point_size = points.size(); + if(m_valid_point_sizes.find(point_size) == m_valid_point_sizes.end()) + return false; + + crypto::Hash hv = crypto::cn_fast_hash(points.data(), points.size() * sizeof(crypto::Hash)); + if(hv != m_targets[point_size]) + { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "CheckpointList verification failed for height " << point_size-1 << + ". Expected hash: " << m_targets[point_size] << + ", Fetched hash: " << hv; + return false; + } + + m_points = std::move(points); + logger(logging::INFO) << "Loaded " << m_points.size() << " checkpoints from local index"; + + save_checkpoints(); + return true; + } + + bool CheckpointList::add_checkpoint_list(uint32_t start_height, std::vector& points) + { + const std::lock_guard lock(m_points_lock); + + if(m_points.size() != start_height) + return true; + + uint32_t point_size = points.size() + m_points.size(); + if(m_valid_point_sizes.find(point_size) == m_valid_point_sizes.end()) + return false; + + /* This copy wouldn't be needed with three funciton hash */ + std::vector new_points(m_points); + new_points.insert(std::end(new_points), std::begin(points), std::end(points)); + + crypto::Hash hv = crypto::cn_fast_hash(new_points.data(), new_points.size() * sizeof(crypto::Hash)); + if(hv != m_targets[point_size]) + { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "CheckpointList verification failed for height " << point_size-1 << + ". Expected hash: " << m_targets[point_size] << + ", Fetched hash: " << hv; + return false; + } + + m_points = std::move(new_points); + logger(logging::INFO) << "Loaded " << points.size() << " checkpoints from p2p, total " << m_points.size(); + + save_checkpoints(); + return true; + } + + bool CheckpointList::load_checkpoints_from_file() + { + std::ifstream file(m_save_file, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + return false; + } + + uint64_t fsize = file.tellg(); + if(!is_fsize_valid(fsize)) { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "Invalid file size" << fsize; + return false; + } + uint32_t point_size = fsize / sizeof(crypto::Hash); + + file.seekg(0, std::ios::beg); + + std::vector points(point_size); + if (!file.read(reinterpret_cast(points.data()), fsize)) { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "error reading file"; + return false; + } + + crypto::Hash hv = crypto::cn_fast_hash(points.data(), points.size() * sizeof(crypto::Hash)); + if(hv != m_targets[point_size]) + { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "CheckpointList verification (from file) failed for height " << point_size-1 + << ". Expected hash: " << m_targets[point_size] + << ", Fetched hash: " << hv; + return false; + } + + const std::lock_guard lock(m_points_lock); + m_points = std::move(points); + logger(logging::INFO) << "Loaded " << m_points.size() << " checkpoints from disk " << m_save_file; + return true; + } + + bool CheckpointList::save_checkpoints() + { + std::ofstream file(m_save_file, std::ios::binary); + + if (!file.is_open()) { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "error opening file for write " << m_save_file; + return false; + } + + file.write(reinterpret_cast(m_points.data()), m_points.size() * sizeof(crypto::Hash)); + file.close(); + + if (!file) { + logger(logging::ERROR) << "<< CheckpointList.cpp << " << "error writing to file " << m_save_file; + return false; + } + + return true; + } +} diff --git a/src/CryptoNoteCore/Core.cpp b/src/CryptoNoteCore/Core.cpp index b9b83c3c6..ee34f853c 100644 --- a/src/CryptoNoteCore/Core.cpp +++ b/src/CryptoNoteCore/Core.cpp @@ -87,10 +87,6 @@ void core::set_cryptonote_protocol(i_cryptonote_protocol* pprotocol) { } } //----------------------------------------------------------------------------------- -void core::set_checkpoints(Checkpoints&& chk_pts) { - m_blockchain.setCheckpoints(std::move(chk_pts)); -} -//----------------------------------------------------------------------------------- void core::init_options(boost::program_options::options_description& /*desc*/) { } diff --git a/src/CryptoNoteCore/Core.h b/src/CryptoNoteCore/Core.h index 11164284e..193e1a84c 100644 --- a/src/CryptoNoteCore/Core.h +++ b/src/CryptoNoteCore/Core.h @@ -83,6 +83,10 @@ namespace cn { virtual bool addMessageQueue(MessageQueue& messageQueue) override; virtual bool removeMessageQueue(MessageQueue& messageQueue) override; + + virtual CheckpointList& getCheckpointList() override { + return m_blockchain.getCheckpointList(); + } uint32_t get_current_blockchain_height(); bool have_block(const crypto::Hash& id) override; @@ -102,6 +106,10 @@ namespace cn { } virtual bool queryBlocks(const std::vector& block_ids, uint64_t timestamp, uint32_t& start_height, uint32_t& current_height, uint32_t& full_offset, std::vector& entries) override; + + virtual std::vector getBlockIds(uint32_t start_height, uint32_t end_height) override { + return m_blockchain.getBlockIds(start_height, end_height); + } virtual bool queryBlocksLite(const std::vector& knownBlockIds, uint64_t timestamp, uint32_t& resStartHeight, uint32_t& resCurrentHeight, uint32_t& resFullOffset, std::vector& entries) override; virtual crypto::Hash getBlockIdByHeight(uint32_t height) override; @@ -118,7 +126,6 @@ namespace cn { uint64_t difficultyAtHeight(uint64_t height); void set_cryptonote_protocol(i_cryptonote_protocol *pprotocol); - void set_checkpoints(Checkpoints &&chk_pts); std::vector getPoolTransactions() override; bool getPoolTransaction(const crypto::Hash &tx_hash, Transaction &transaction) override; diff --git a/src/CryptoNoteCore/Currency.cpp b/src/CryptoNoteCore/Currency.cpp index b493e9821..a46051c17 100644 --- a/src/CryptoNoteCore/Currency.cpp +++ b/src/CryptoNoteCore/Currency.cpp @@ -1392,6 +1392,7 @@ namespace cn blockIndexesFileName(parameters::CRYPTONOTE_BLOCKINDEXES_FILENAME); txPoolFileName(parameters::CRYPTONOTE_POOLDATA_FILENAME); blockchinIndicesFileName(parameters::CRYPTONOTE_BLOCKCHAIN_INDICES_FILENAME); + checkpointFileName(parameters::CRYPTONOTE_CHECKPOINT_FILENAME); testnet(false); } diff --git a/src/CryptoNoteCore/Currency.h b/src/CryptoNoteCore/Currency.h index a81f91c43..d70b1d402 100644 --- a/src/CryptoNoteCore/Currency.h +++ b/src/CryptoNoteCore/Currency.h @@ -148,6 +148,7 @@ namespace cn const std::string &blockIndexesFileName() const { return m_blockIndexesFileName; } const std::string &txPoolFileName() const { return m_txPoolFileName; } const std::string &blockchinIndicesFileName() const { return m_blockchinIndicesFileName; } + const std::string &checkpointFileName() const { return m_checkpointFileName; } bool isTestnet() const { return m_testnet; } @@ -290,6 +291,7 @@ namespace cn std::string m_blockIndexesFileName; std::string m_txPoolFileName; std::string m_blockchinIndicesFileName; + std::string m_checkpointFileName; static const std::vector REWARD_INCREASING_FACTOR; @@ -634,6 +636,11 @@ namespace cn m_currency.m_blockchinIndicesFileName = val; return *this; } + CurrencyBuilder &checkpointFileName(const std::string &val) + { + m_currency.m_checkpointFileName = val; + return *this; + } CurrencyBuilder &genesisCoinbaseTxHex(const std::string &val) { @@ -694,4 +701,4 @@ namespace cn } }; -} // namespace cn \ No newline at end of file +} // namespace cn diff --git a/src/CryptoNoteCore/ICore.h b/src/CryptoNoteCore/ICore.h index 91b883c14..faeb5ca5a 100644 --- a/src/CryptoNoteCore/ICore.h +++ b/src/CryptoNoteCore/ICore.h @@ -27,6 +27,7 @@ struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS_response; struct NOTIFY_RESPONSE_GET_OBJECTS_request; struct NOTIFY_REQUEST_GET_OBJECTS_request; +class CheckpointList; class Currency; class IBlock; class ICoreObserver; @@ -65,6 +66,7 @@ class ICore { virtual void on_synchronized() = 0; virtual size_t addChain(const std::vector& chain) = 0; + virtual CheckpointList& getCheckpointList() = 0; virtual void get_blockchain_top(uint32_t& height, crypto::Hash& top_id) = 0; virtual std::vector findBlockchainSupplement(const std::vector& remoteBlockIds, size_t maxCount, uint32_t& totalBlockCount, uint32_t& startBlockIndex) = 0; @@ -83,6 +85,7 @@ class ICore { std::vector& deletedTxsIds) = 0; virtual bool queryBlocks(const std::vector& block_ids, uint64_t timestamp, uint32_t& start_height, uint32_t& current_height, uint32_t& full_offset, std::vector& entries) = 0; + virtual std::vector getBlockIds(uint32_t start_height, uint32_t end_height) = 0; virtual bool queryBlocksLite(const std::vector& block_ids, uint64_t timestamp, uint32_t& start_height, uint32_t& current_height, uint32_t& full_offset, std::vector& entries) = 0; diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolDefinitions.h b/src/CryptoNoteProtocol/CryptoNoteProtocolDefinitions.h index b9f626115..77244e9d7 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolDefinitions.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolDefinitions.h @@ -225,6 +225,9 @@ namespace cn typedef NOTIFY_NEW_LITE_BLOCK_request request; }; + /************************************************************************/ + /* */ + /************************************************************************/ struct NOTIFY_MISSING_TXS_request { crypto::Hash blockHash; @@ -244,5 +247,46 @@ namespace cn const static int ID = BC_COMMANDS_POOL_BASE + 10; typedef NOTIFY_MISSING_TXS_request request; }; + + /************************************************************************/ + /* */ + /************************************************************************/ + struct NOTIFY_REQUEST_CHECKPOINT_LIST_request + { + crypto::Hash target_hash; + uint32_t start_height; + uint32_t end_height; + + void serialize(ISerializer &s) + { + KV_MEMBER(target_hash) + KV_MEMBER(start_height) + KV_MEMBER(end_height) + } + }; + + struct NOTIFY_REQUEST_CHECKPOINT_LIST + { + const static int ID = BC_COMMANDS_POOL_BASE + 11; + typedef NOTIFY_REQUEST_CHECKPOINT_LIST_request request; + }; + + struct NOTIFY_RESPONSE_CHECKPOINT_LIST_request + { + uint32_t list_start_height; + std::vector checkpoint_list; + + void serialize(ISerializer &s) + { + KV_MEMBER(list_start_height) + serializeAsBinary(checkpoint_list, "checkpoint_list", s); + } + }; + + struct NOTIFY_RESPONSE_CHECKPOINT_LIST + { + const static int ID = BC_COMMANDS_POOL_BASE + 12; + typedef NOTIFY_RESPONSE_CHECKPOINT_LIST_request request; + }; } // namespace cn diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index beea9661f..c3c17156c 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -20,6 +20,7 @@ #include "CryptoNoteCore/Currency.h" #include "CryptoNoteCore/VerificationContext.h" #include "P2p/LevinProtocol.h" +#include "CryptoNoteCore/CheckpointList.h" using namespace logging; using namespace common; @@ -50,6 +51,7 @@ CryptoNoteProtocolHandler::CryptoNoteProtocolHandler(const Currency ¤cy, p m_core(rcore), m_synchronized(false), m_stop(false), + m_last_checkpoint_req(0), m_observedHeight(0), m_peersCount(0), logger(log, "protocol"), @@ -193,10 +195,10 @@ bool CryptoNoteProtocolHandler::process_payload_sync_data(const CORE_SYNC_DATA & { int64_t diff = static_cast(hshd.current_height) - static_cast(get_current_blockchain_height()); - logger(diff >= 0 ? (is_inital ? logging::INFO : DEBUGGING) : logging::TRACE) << context << "Unknown top block: " << get_current_blockchain_height() << " -> " << hshd.current_height - << std::endl - - << "Synchronization started"; + logger(diff >= 0 ? (is_inital ? logging::INFO : DEBUGGING) : logging::TRACE) << context + << "Unknown top block: " << get_current_blockchain_height() << " -> " << hshd.current_height + << std::endl + << "Synchronization started"; logger(DEBUGGING) << "Remote top block height: " << hshd.current_height << ", id: " << hshd.top_id; //let the socket to send response to handshake, but request callback, to let send request data after response @@ -213,6 +215,29 @@ bool CryptoNoteProtocolHandler::process_payload_sync_data(const CORE_SYNC_DATA & m_observerManager.notify(&ICryptoNoteProtocolObserver::peerCountUpdated, m_peersCount.load()); } + if (context.version < cn::P2P_CHECKPOINT_LIST_VERSION || context.m_checkpoints_not_in_sync || context.m_active_checkpoint_req) + return true; + + uint64_t time_now = time(nullptr); + uint64_t last_request = time_now - m_last_checkpoint_req.load(); + if (last_request > P2P_CHECKPOINT_LIST_RE_REQUEST) + return true; + + auto cl_status = m_core.getCheckpointList().get_incomplete_checkpoint_target(); + if ( cl_status.target_hash != NULL_HASH ) + { + NOTIFY_REQUEST_CHECKPOINT_LIST::request req = boost::value_initialized(); + req.target_hash = cl_status.target_hash; + req.start_height = cl_status.start_height; + req.end_height = cl_status.end_height; + + if (!post_notify(*m_p2p, req, context)) + return false; + + m_last_checkpoint_req = time_now; + context.m_active_checkpoint_req = true; + } + return true; } @@ -259,6 +284,8 @@ int CryptoNoteProtocolHandler::handleCommand(bool is_notify, int command, const HANDLE_NOTIFY(NOTIFY_NEW_TRANSACTIONS, &CryptoNoteProtocolHandler::handle_notify_new_transactions) HANDLE_NOTIFY(NOTIFY_REQUEST_GET_OBJECTS, &CryptoNoteProtocolHandler::handle_request_get_objects) HANDLE_NOTIFY(NOTIFY_RESPONSE_GET_OBJECTS, &CryptoNoteProtocolHandler::handle_response_get_objects) + HANDLE_NOTIFY(NOTIFY_REQUEST_CHECKPOINT_LIST, &CryptoNoteProtocolHandler::handle_request_checkpoint_list) + HANDLE_NOTIFY(NOTIFY_RESPONSE_CHECKPOINT_LIST, &CryptoNoteProtocolHandler::handle_response_checkpoint_list) HANDLE_NOTIFY(NOTIFY_REQUEST_CHAIN, &CryptoNoteProtocolHandler::handle_request_chain) HANDLE_NOTIFY(NOTIFY_RESPONSE_CHAIN_ENTRY, &CryptoNoteProtocolHandler::handle_response_chain_entry) HANDLE_NOTIFY(NOTIFY_REQUEST_TX_POOL, &CryptoNoteProtocolHandler::handle_request_tx_pool) @@ -1121,4 +1148,46 @@ int CryptoNoteProtocolHandler::doPushLiteBlock(NOTIFY_NEW_LITE_BLOCK::request ar return 1; } +int CryptoNoteProtocolHandler::handle_request_checkpoint_list(int command, NOTIFY_REQUEST_CHECKPOINT_LIST::request& arg, CryptoNoteConnectionContext& context) +{ + NOTIFY_RESPONSE_CHECKPOINT_LIST::request rsp; + rsp.list_start_height = arg.start_height; + rsp.checkpoint_list = m_core.getBlockIds(arg.start_height, arg.end_height); + + crypto::Hash hv = crypto::cn_fast_hash(rsp.checkpoint_list.data(), rsp.checkpoint_list.size() * sizeof(crypto::Hash)); + if ( hv != arg.target_hash || rsp.checkpoint_list.size() == 0) + { + logger(logging::ERROR) << context << " peer requested different hash for start_height " << arg.start_height + << " end_height " << arg.end_height << " checkpoint list size " << rsp.checkpoint_list.size() + << " I have hash " << hv << " peer wants " << arg.target_hash; + + rsp.checkpoint_list.clear(); + rsp.list_start_height = 0; + context.m_checkpoints_not_in_sync = true; + } + + if (!post_notify(*m_p2p, rsp, context)) + context.m_state = CryptoNoteConnectionContext::state_shutdown; + return 1; +} + +int CryptoNoteProtocolHandler::handle_response_checkpoint_list(int command, NOTIFY_RESPONSE_CHECKPOINT_LIST::request& arg, CryptoNoteConnectionContext& context) +{ + if( !context.m_active_checkpoint_req ) + { + context.m_state = CryptoNoteConnectionContext::state_shutdown; + return 1; + } + + m_last_checkpoint_req = 0; + context.m_active_checkpoint_req = false; + + if( arg.checkpoint_list.size() == 0 || !m_core.getCheckpointList().add_checkpoint_list(arg.list_start_height, arg.checkpoint_list) ) + { + context.m_checkpoints_not_in_sync = true; + } + + return 1; +} + }; // namespace cn diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h index ccb1d1bfa..7788817e5 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h @@ -80,6 +80,8 @@ namespace cn int handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, CryptoNoteConnectionContext& context); int handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, CryptoNoteConnectionContext& context); int handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, CryptoNoteConnectionContext& context); + int handle_request_checkpoint_list(int command, NOTIFY_REQUEST_CHECKPOINT_LIST::request& arg, CryptoNoteConnectionContext& context); + int handle_response_checkpoint_list(int command, NOTIFY_RESPONSE_CHECKPOINT_LIST::request& arg, CryptoNoteConnectionContext& context); int handle_request_chain(int command, NOTIFY_REQUEST_CHAIN::request& arg, CryptoNoteConnectionContext& context); int handle_response_chain_entry(int command, NOTIFY_RESPONSE_CHAIN_ENTRY::request& arg, CryptoNoteConnectionContext& context); int handle_request_tx_pool(int command, NOTIFY_REQUEST_TX_POOL::request &arg, CryptoNoteConnectionContext &context); @@ -111,7 +113,9 @@ namespace cn IP2pEndpoint* m_p2p; std::atomic m_synchronized; std::atomic m_stop; - std::recursive_mutex m_sync_lock; + std::recursive_mutex m_sync_lock; + + std::atomic m_last_checkpoint_req; mutable std::mutex m_observedHeightMutex; uint32_t m_observedHeight; diff --git a/src/Daemon/Daemon.cpp b/src/Daemon/Daemon.cpp index d805957ad..ef7387e2a 100644 --- a/src/Daemon/Daemon.cpp +++ b/src/Daemon/Daemon.cpp @@ -17,7 +17,7 @@ #include "Common/PathTools.h" #include "crypto/hash.h" #include "CryptoNoteConfig.h" -#include "CryptoNoteCore/Checkpoints.h" +#include "CryptoNoteCore/CheckpointList.h" #include "CryptoNoteCore/Core.h" #include "CryptoNoteCore/CoreConfig.h" #include "CryptoNoteCore/CryptoNoteTools.h" @@ -226,12 +226,6 @@ int main(int argc, char* argv[]) cn::Currency currency = currencyBuilder.currency(); cn::core ccore(currency, nullptr, logManager, vm["enable-blockchain-indexes"].as(), vm["enable-autosave"].as()); - cn::Checkpoints checkpoints(logManager); - checkpoints.set_testnet(coreConfig.testnet); - checkpoints.load_checkpoints(); - checkpoints.load_checkpoints_from_dns(); - ccore.set_checkpoints(std::move(checkpoints)); - NetNodeConfig netNodeConfig; netNodeConfig.init(vm); netNodeConfig.setTestnet(coreConfig.testnet); diff --git a/src/P2p/ConnectionContext.h b/src/P2p/ConnectionContext.h index 7ac24e29d..fa4f3d76a 100644 --- a/src/P2p/ConnectionContext.h +++ b/src/P2p/ConnectionContext.h @@ -27,6 +27,8 @@ struct CryptoNoteConnectionContext { uint32_t m_remote_port = 0; bool m_is_income = false; time_t m_started = 0; + bool m_checkpoints_not_in_sync = false; + bool m_active_checkpoint_req = false; enum state { state_befor_handshake = 0, //default state From 52950f0ce6166a7650f3fcb120d0e88c6042882f Mon Sep 17 00:00:00 2001 From: AxVultis Date: Sun, 16 Jun 2024 16:21:21 +0200 Subject: [PATCH 04/56] Copyright Update 2024 --- COPYING.md | 24 ------------------------ COPYRIGHT | 24 ------------------------ LICENSE | 7 +++---- README.md | 2 +- 4 files changed, 4 insertions(+), 53 deletions(-) delete mode 100644 COPYING.md delete mode 100644 COPYRIGHT diff --git a/COPYING.md b/COPYING.md deleted file mode 100644 index aa6404ce4..000000000 --- a/COPYING.md +++ /dev/null @@ -1,24 +0,0 @@ -MIT License -Distributed under the MIT/X11 software license http://www.opensource.org/licenses/mit-license.php - -Copyright (c) 2017-2023 Conceal Community -Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -Copyright (c) 2018-2023 Conceal Network & Conceal Devs - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/COPYRIGHT b/COPYRIGHT deleted file mode 100644 index aa6404ce4..000000000 --- a/COPYRIGHT +++ /dev/null @@ -1,24 +0,0 @@ -MIT License -Distributed under the MIT/X11 software license http://www.opensource.org/licenses/mit-license.php - -Copyright (c) 2017-2023 Conceal Community -Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -Copyright (c) 2018-2023 Conceal Network & Conceal Devs - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/LICENSE b/LICENSE index aa6404ce4..f9f447e7a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,8 @@ MIT License -Distributed under the MIT/X11 software license http://www.opensource.org/licenses/mit-license.php -Copyright (c) 2017-2023 Conceal Community +Copyright (c) 2011-2017 The Cryptonote developers Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -Copyright (c) 2018-2023 Conceal Network & Conceal Devs +Copyright (c) 2018-2024 Conceal Network & Conceal Devs Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -21,4 +20,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index b34645117..61b736a74 100644 --- a/README.md +++ b/README.md @@ -167,4 +167,4 @@ If the build is successful the binaries will be located in the `src` folder. Special thanks goes out to the developers from Cryptonote, Bytecoin, Ryo, Monero, Forknote, TurtleCoin, Karbo and Masari. -Copyright (c) 2017-2023 Conceal Community, Conceal Network & Conceal Devs \ No newline at end of file +Copyright (c) 2017-2024 Conceal Community, Conceal Network & Conceal Devs \ No newline at end of file From 55dd43f78dfdb455c86634784a0c3009a54fdec2 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 20 Jan 2025 11:54:15 -0500 Subject: [PATCH 05/56] update parse extra handling --- .gitmodules | 4 ++++ cryptonote | 1 + src/CryptoNoteCore/TransactionExtra.cpp | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 cryptonote diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..803bf432e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "cryptonote"] + path = cryptonote + url = git@github.com:Acktarius/cryptonote.git + branch = ack/fix-paimentId diff --git a/cryptonote b/cryptonote new file mode 160000 index 000000000..eb6f0582c --- /dev/null +++ b/cryptonote @@ -0,0 +1 @@ +Subproject commit eb6f0582c6296f7193201ea9d575c8c669413c49 diff --git a/src/CryptoNoteCore/TransactionExtra.cpp b/src/CryptoNoteCore/TransactionExtra.cpp index 9f8118139..4510acab5 100644 --- a/src/CryptoNoteCore/TransactionExtra.cpp +++ b/src/CryptoNoteCore/TransactionExtra.cpp @@ -310,7 +310,22 @@ namespace cn bool parsePaymentId(const std::string &paymentIdString, Hash &paymentId) { - return common::podFromHex(paymentIdString, paymentId); + // If input looks like transaction extra data (35 bytes) + if (paymentIdString.length() == 35) { + std::vector extraNonce; + extraNonce.resize(paymentIdString.length() - 2); // Skip TX_EXTRA_NONCE tag + memcpy(extraNonce.data(), paymentIdString.data() + 2, paymentIdString.length() - 2); + + if (getPaymentIdFromTransactionExtraNonce(extraNonce, paymentId)) { + return true; + } + } + // Try parsing as hex string + if (paymentIdString.length() == 64) { + return common::podFromHex(paymentIdString, paymentId); + } + + return false; } bool createTxExtraWithPaymentId(const std::string &paymentIdString, std::vector &extra) From c44924d7148bd4eeacd25bf9ae7f7e405c0d3d50 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 20 Jan 2025 14:12:27 -0500 Subject: [PATCH 06/56] git clean submodule --- .gitmodules | 4 ---- cryptonote | 1 - 2 files changed, 5 deletions(-) delete mode 100644 .gitmodules delete mode 160000 cryptonote diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 803bf432e..000000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "cryptonote"] - path = cryptonote - url = git@github.com:Acktarius/cryptonote.git - branch = ack/fix-paimentId diff --git a/cryptonote b/cryptonote deleted file mode 160000 index eb6f0582c..000000000 --- a/cryptonote +++ /dev/null @@ -1 +0,0 @@ -Subproject commit eb6f0582c6296f7193201ea9d575c8c669413c49 From 05d1d3b76979d3d1b13520cc284e48fc95a378a1 Mon Sep 17 00:00:00 2001 From: acktarius Date: Sun, 30 Nov 2025 20:23:35 -0500 Subject: [PATCH 07/56] log connections --- .../CryptoNoteProtocolHandler.cpp | 63 +++++++++++++++---- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 00d6d2ea3..26e4159ea 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1329,6 +1330,8 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE return 1; } + logger(INFO) << context << " Received chunk hash response from peer " << peer_id << " for chunk " << arg.chunk_index; + // Store the response in pending chunk hashes map for the consensus mechanism // NOTE: NULL_HASH is a valid response (means peer doesn't have this chunk) { @@ -1367,7 +1370,7 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe // Log all available peer IDs for debugging logger(DEBUGGING) << "Available peer IDs:"; m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t id) { - logger(DEBUGGING) << " Peer ID: " << id << ", connection_id: " << ctx.m_connection_id + logger(DEBUGGING) << " Peer ID: " << id << " " << ctx << ", state: " << get_protocol_state_string(ctx.m_state) << ", version: " << static_cast(ctx.version); }); @@ -1385,18 +1388,20 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe req.chunk_index = chunk_index; logger(INFO) << "Requesting chunk hash for chunk " << chunk_index << " from peer " << peer_id + << " " << *peer_context << " (connection_id: " << peer_context->m_connection_id << ", state: " << get_protocol_state_string(peer_context->m_state) << ", version: " << static_cast(peer_context->version) << ")"; bool sent = post_notify(*m_p2p, req, *peer_context); if (!sent) { - logger(WARNING) << "Failed to send chunk hash request to peer " << peer_id << " for chunk " << chunk_index; + logger(WARNING) << "Failed to send chunk hash request to peer " << peer_id << " " << *peer_context + << " for chunk " << chunk_index; return NULL_HASH; } logger(INFO) << "Successfully sent chunk hash request for chunk " << chunk_index - << " to peer " << peer_id << " (waiting for response...)"; + << " to peer " << peer_id << " " << *peer_context << " (waiting for response...)"; // Wait for response (with timeout) // In a real implementation, this should use async/await or a callback mechanism @@ -1414,13 +1419,14 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe if (it != m_pending_chunk_hashes.end()) { crypto::Hash result = it->second; m_pending_chunk_hashes.erase(it); - logger(INFO) << "Received chunk hash response from peer " << peer_id + logger(INFO) << "Received chunk hash response from peer " << peer_id << " " << *peer_context << " for chunk " << chunk_index << ": " << result; return result; } } - logger(logging::WARNING) << "Timeout waiting for chunk hash response from peer " << peer_id << " for chunk " << chunk_index; + logger(logging::WARNING) << "Timeout waiting for chunk hash response from peer " << peer_id + << " " << *peer_context << " for chunk " << chunk_index; return NULL_HASH; } @@ -1515,10 +1521,19 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() std::vector eligible_peers; std::map peer_network_16; // peer_id -> /16 network + // First, log all connected peers for debugging + uint32_t total_connected = 0; + m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { + total_connected++; + }); + logger(INFO) << "Checking " << total_connected << " connected peer(s) for chunk validation eligibility"; + m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { // Only consider peers that support chunk-based checkpoints if (ctx.version < cn::P2P_CHECKPOINT_LIST_VERSION) { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: version " + << static_cast(ctx.version) << " < " << cn::P2P_CHECKPOINT_LIST_VERSION; return; // Skip old version peers } @@ -1527,6 +1542,8 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() if (ctx.m_state != CryptoNoteConnectionContext::state_normal && ctx.m_state != CryptoNoteConnectionContext::state_synchronizing) { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: state " + << get_protocol_state_string(ctx.m_state) << " (need normal or synchronizing)"; return; // Skip peers that aren't in a usable state } @@ -1536,6 +1553,7 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() time_t connection_duration = time_now - ctx.m_started; if (connection_duration < 0) { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: invalid connection time"; return; // Invalid connection time } @@ -1544,10 +1562,16 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() // Check if peer meets minimum uptime requirement if (peer_uptime_blocks < min_uptime_blocks) { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: uptime " + << peer_uptime_blocks << " blocks < " << min_uptime_blocks << " blocks required"; return; // Peer doesn't meet uptime requirement } // Peer is eligible + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " is ELIGIBLE: version " + << static_cast(ctx.version) << ", state " + << get_protocol_state_string(ctx.m_state) << ", uptime " + << peer_uptime_blocks << " blocks"; eligible_peers.push_back(peer_id); peer_network_16[peer_id] = CheckpointList::get_network_16(ctx.m_remote_ip); }); @@ -1556,6 +1580,9 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() { logger(INFO) << "No eligible peers for chunk validation (need uptime > " << min_uptime_blocks << " blocks, version 2+, and in normal/synchronizing state)"; + logger(INFO) << "Note: Only ACTIVE CONNECTIONS are considered, not peers in peerlist. " + << "Use 'print_cn' command to see active connections. " + << "To force connection to a peer, use --add-priority-node :"; logger(INFO) << "Chunk validation will be retried when eligible peers become available"; return false; } @@ -1606,7 +1633,16 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() crypto::Hash peer_hash = request_chunk_hash_from_peer(peer_id, validating_chunk); if (peer_hash == NULL_HASH) { - logger(INFO) << "Peer " << peer_id << " does not have chunk " << validating_chunk + // Find peer context to log IP address + std::string peer_info = "peer " + std::to_string(peer_id); + m_p2p->for_each_connection([&peer_info, peer_id](const CryptoNoteConnectionContext& ctx, uint64_t id) { + if (id == peer_id) { + std::ostringstream oss; + oss << ctx; + peer_info = "peer " + std::to_string(peer_id) + " " + oss.str(); + } + }); + logger(INFO) << peer_info << " does not have chunk " << validating_chunk << " in memory (returned NULL_HASH)"; } return peer_hash; @@ -1642,9 +1678,11 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() { logger(INFO) << "Chunk " << chunk_index << " validation: No peers have this chunk in memory yet. " - << "Will retry validation once peers create this chunk."; + << "Will retry validation once peers create this chunk. " + << "Stopping validation - no point to add subsequent chunks to checkpoint.dat " + << "if chunk " << chunk_index << " is not validated first (file must be sequential)."; // Don't rollback - peers just need to create the chunk first - continue; // Skip to next chunk or wait + break; // Stop - file must be sequential, no gaps allowed } if (result.consensus_reached) @@ -1674,9 +1712,11 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() << (chunk_index > 0 ? chunk_index - 1 : 0) << " boundary."; // Calculate rollback height - // chunk[0]: blocks 0 to chunk_size (inclusive) - // chunk[1]: blocks (chunk_size + 1) to (2 * chunk_size) (inclusive) - // So chunk N ends at: (N + 1) * chunk_size + // SIMPLIFIED: Block 0 (genesis) is NOT in any chunk + // chunk[0]: blocks 1 to chunk_size + // chunk[1]: blocks (chunk_size + 1) to (2 * chunk_size) + // chunk[N]: blocks (N * chunk_size + 1) to ((N + 1) * chunk_size) + // // Rollback to end of previous chunk (chunk_index - 1) uint32_t chunk_size = m_core.getCheckpointList().get_chunk_size(); uint32_t rollback_height; @@ -1688,6 +1728,7 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() { // Rollback to end of previous chunk // Previous chunk (chunk_index - 1) ends at: chunk_index * chunk_size + // Example: If chunk 38 fails, rollback to: 38 * 10000 = 380,000 (end of chunk 37) rollback_height = chunk_index * chunk_size; } From 73aaa5147450c076572399b07d7af0526d7c156a Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 07:39:33 -0500 Subject: [PATCH 08/56] Refined checkpoint * strategy with priority order and validation of checkpoint hashes --- src/CryptoNoteCore/Blockchain.cpp | 59 ++++++++++- src/CryptoNoteCore/CheckpointList.h | 3 + src/CryptoNoteCore/CheckpointsList.cpp | 137 ++++++++++++++++++++++--- 3 files changed, 178 insertions(+), 21 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 8442ad104..845bfd218 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -760,10 +760,61 @@ namespace cn logger(INFO, BRIGHT_GREEN) << "Successfully generated " << chunkCount << " checkpoint chunks from CryptoNoteConfig.h (covers up to height " << coveredHeight << ")"; - // Step 2: Check if DNS checkpoints extend beyond CryptoNoteConfig.h - // If so, try to extend chunks up to the highest DNS checkpoint - // (This will be handled by the existing logic that creates missing chunks beyond hardcoded checkpoints) - // The chunks will use priority order (CryptoNoteConfig.h > DNS > blockchain.dat) during generation + // Step 2: Create chunks beyond hardcoded checkpoints up to current blockchain height + // These chunks will be stored in memory and validated via P2P consensus + if (currentHeight > greatestTargetHeight) + { + uint32_t chunk_size = m_checkpoints.get_chunk_size(); + uint32_t last_hardcoded_chunk_index = (greatestTargetHeight - 1) / chunk_size; + uint32_t current_chunk_index = (currentHeight - 1) / chunk_size; + + logger(INFO) << "Creating chunks " << (last_hardcoded_chunk_index + 1) + << " to " << current_chunk_index + << " (beyond hardcoded checkpoints, up to current height " << currentHeight << ")"; + + // Create chunks beyond hardcoded checkpoints (these need P2P validation) + // Use same logic as existing code path (lines 915-952) for consistency + for (uint32_t chunk_idx = last_hardcoded_chunk_index + 1; chunk_idx <= current_chunk_index; chunk_idx++) + { + // SIMPLIFIED: All chunks are uniform (block 0 excluded) + uint32_t chunk_start_height = chunk_idx * chunk_size + 1; + uint32_t chunk_end_height = (chunk_idx + 1) * chunk_size; + + // Only create if we have all blocks for this chunk + if (currentHeight >= chunk_end_height) + { + // Check if chunk already exists in memory (from previous session) + crypto::Hash existing_chunk_hash = m_checkpoints.get_chunk_hash(chunk_idx); + if (existing_chunk_hash == NULL_HASH) + { + if (m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) + { + logger(INFO) << "Created checkpoint chunk " << chunk_idx + << " (blocks " << chunk_start_height << "-" << chunk_end_height + << ") - stored in memory, awaiting P2P validation"; + // NOTE: We do NOT call add_verified_chunk_to_file() here + // These chunks need peer consensus before being saved to checkpoint.dat + } + else + { + logger(WARNING) << "Failed to create chunk " << chunk_idx << " - blockchain mismatch detected"; + break; + } + } + else + { + logger(DEBUGGING) << "Chunk " << chunk_idx << " already exists in memory (from previous session), awaiting P2P validation"; + } + } + else + { + logger(DEBUGGING) << "Skipping chunk " << chunk_idx + << " - not enough blocks (have " << currentHeight + << ", need " << chunk_end_height << ")"; + break; + } + } + } logger(INFO) << "Initial setup complete. Chunks beyond CryptoNoteConfig.h checkpoints will be validated via P2P consensus."; } diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index 5ddfb13a7..ef7676446 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -464,6 +464,9 @@ namespace cn bool save_checkpoints_legacy(); bool load_checkpoints_from_file_legacy(); + // Migration: Convert legacy format (individual block hashes) to chunked format + bool convert_legacy_to_chunked_format(); + bool add_checkpoint_target(uint32_t height, const std::string &hash_str); // Helper: Calculate which chunk a block height belongs to diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index ef7cfebc2..6aa4c8def 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -447,21 +447,7 @@ namespace cn { // VALIDATION: Verify that blockchain.dat matches the CryptoNoteConfig.h checkpoint // (We validate against blockchain.dat, not the potentially overwritten DNS value) - // Get the original blockchain hash by checking if we already applied DNS const crypto::Hash& config_hash = checkpoint.second; - const crypto::Hash& current_hash = chunk_block_ids[index_in_chunk]; - - // If this height was already overwritten by DNS, we need to validate against blockchain.dat - // For now, we validate against the current value (which might be DNS or blockchain.dat) - // The important thing is that blockchain.dat must match the config checkpoint - // We'll validate this by checking if current_hash matches config_hash OR if we need to check blockchain.dat - - // Actually, we should validate against blockchain.dat directly - // But we've already overwritten chunk_block_ids with DNS values - // So we need to track original blockchain.dat values, OR re-fetch, OR validate before overwriting - // Simplest: validate that current value (blockchain.dat or DNS) matches config - // If DNS was applied, current_hash is DNS hash, which should match blockchain.dat (we validated above) - // So if config_hash != current_hash, we need to check if blockchain.dat matches config_hash // Re-fetch the original blockchain.dat hash for validation std::vector original_block_ids = getBlockIdsFunc(checkpoint_height, 1); @@ -1180,9 +1166,127 @@ namespace cn { m_points = std::move(points); logger(INFO) << "Loaded " << m_points.size() << " checkpoints from disk (legacy format) " << m_save_file; - // TODO: Convert legacy format to chunked format automatically - // For now, we keep both formats during transition + // Automatically convert legacy format to chunked format (only for version 2+) + // Version 1 nodes continue using legacy format + if (cn::P2P_CURRENT_VERSION >= cn::P2P_CHECKPOINT_LIST_VERSION) { + if (convert_legacy_to_chunked_format()) { + logger(INFO) << "Successfully converted legacy checkpoint format to chunked format"; + + // Backup old file before saving new format + std::string backup_file = m_save_file + ".bckv1"; + std::ifstream src(m_save_file, std::ios::binary); + if (src.good()) { + std::ofstream dst(backup_file, std::ios::binary); + if (dst.good()) { + dst << src.rdbuf(); + logger(INFO) << "Backed up legacy checkpoint file to " << backup_file; + } else { + logger(WARNING) << "Failed to create backup file " << backup_file; + } + } + + // Save in new chunked format (overwrites old file) + if (save_checkpoints()) { + logger(INFO) << "Migration complete: saved chunked format, legacy format backed up to " << backup_file; + } else { + logger(ERROR) << "Migration failed: could not save chunked format"; + return true; // Still return true - legacy format is loaded + } + } else { + logger(WARNING) << "Failed to convert legacy format to chunked format - keeping legacy format"; + } + } else { + logger(INFO) << "Node is version 1 - keeping legacy checkpoint format (migration only for version 2+)"; + } + + return true; + } + + /** + * Convert legacy format (individual block hashes) to chunked format + * + * Legacy format: m_points[i] = block hash at height i (i=0 is genesis, i=1 is block 1, etc.) + * Chunked format: m_chunks[j] = hash of all block hashes in chunk j (chunks exclude block 0) + * + * Conversion: + * - Skip block 0 (genesis) - chunks don't include it + * - chunk[0] = hash(m_points[1] || m_points[2] || ... || m_points[chunk_size]) + * - chunk[1] = hash(m_points[chunk_size+1] || m_points[chunk_size+2] || ... || m_points[2*chunk_size]) + * - etc. + */ + bool CheckpointList::convert_legacy_to_chunked_format() + { + std::lock_guard points_lock(m_points_lock); + + if (m_points.empty()) { + logger(WARNING) << "Cannot convert: legacy format is empty"; + return false; + } + + // Legacy format: m_points[0] = block 0 (genesis), m_points[1] = block 1, etc. + // Chunks exclude block 0, so we start from m_points[1] + uint32_t total_blocks = static_cast(m_points.size()); + if (total_blocks < 2) { + logger(WARNING) << "Cannot convert: need at least block 0 and block 1 (have " << total_blocks << " blocks)"; + return false; + } + + // Number of blocks to convert (excluding block 0/genesis) + uint32_t blocks_to_convert = total_blocks - 1; + + // Calculate number of chunks + uint32_t num_chunks = blocks_to_convert / m_chunk_size; + + if (num_chunks == 0) { + logger(WARNING) << "Cannot convert: not enough blocks for even one chunk (have " << blocks_to_convert + << " blocks, need " << m_chunk_size << " per chunk)"; + return false; + } + + logger(INFO) << "Converting legacy format to chunked format: " << total_blocks + << " blocks (" << blocks_to_convert << " excluding genesis) -> " + << num_chunks << " chunks"; + + // Convert to chunks + std::lock_guard chunks_lock(m_chunks_lock); + m_chunks.clear(); + m_chunks.reserve(num_chunks); + + for (uint32_t chunk_index = 0; chunk_index < num_chunks; chunk_index++) { + uint32_t start_block_index = chunk_index * m_chunk_size + 1; // +1 to skip block 0 + uint32_t end_block_index = std::min(start_block_index + m_chunk_size, total_blocks); + uint32_t blocks_in_chunk = end_block_index - start_block_index; + + if (blocks_in_chunk != m_chunk_size && chunk_index < num_chunks - 1) { + // Last chunk can be partial, but intermediate chunks must be full + logger(ERROR) << "Invalid chunk " << chunk_index << ": expected " << m_chunk_size + << " blocks, got " << blocks_in_chunk; + return false; + } + + // Extract block hashes for this chunk + std::vector chunk_block_ids; + chunk_block_ids.reserve(blocks_in_chunk); + for (uint32_t i = start_block_index; i < end_block_index; i++) { + chunk_block_ids.push_back(m_points[i]); + } + + // Calculate chunk hash + crypto::Hash chunk_hash = crypto::cn_fast_hash( + chunk_block_ids.data(), + chunk_block_ids.size() * sizeof(crypto::Hash) + ); + + m_chunks.push_back(chunk_hash); + } + + // Mark all converted chunks as confirmed (they came from validated legacy format) + m_confirmed_chunks.clear(); + for (uint32_t i = 0; i < m_chunks.size(); i++) { + m_confirmed_chunks.insert(i); + } + logger(INFO) << "Successfully converted " << num_chunks << " chunks from legacy format"; return true; } @@ -1499,7 +1603,6 @@ namespace cn { for (const auto& checkpoint_entry : checkpoints_to_verify) { uint32_t checkpoint_height = checkpoint_entry.first; - const crypto::Hash& expected_checkpoint_hash = checkpoint_entry.second.first; const std::string& source = checkpoint_entry.second.second; // SIMPLIFIED: Find which chunk this checkpoint belongs to From cc0a5cf48d1f96f5ba49adca64ccd9b6760456a1 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 08:39:35 -0500 Subject: [PATCH 09/56] rollback strategy * rollback only after second concesus failed twice with the same hash --- src/CryptoNoteCore/CheckpointList.h | 2 + src/CryptoNoteCore/CheckpointsList.cpp | 137 ++++++++++++++++-- .../CryptoNoteProtocolHandler.cpp | 41 ++++-- 3 files changed, 155 insertions(+), 25 deletions(-) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index ef7676446..5f5bac8b0 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -174,6 +174,8 @@ namespace cn uint32_t agreements_first_attempt; // Number of agreements in first attempt uint32_t agreements_second_attempt; // Number of agreements in second attempt (if retried) bool used_second_chance; // true if second attempt was made + crypto::Hash consensus_hash_first_attempt; // Hash that M/K peers agreed on in attempt 1 (if different from local, NULL_HASH otherwise) + crypto::Hash consensus_hash_second_attempt; // Hash that M/K peers agreed on in attempt 2 (if different from local, NULL_HASH otherwise) }; ConsensusResult verify_chunk_with_peer_consensus( uint32_t chunk_index, diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 6aa4c8def..4b402e7fd 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -1819,6 +1819,8 @@ namespace cn { result.agreements_first_attempt = 0; result.agreements_second_attempt = 0; result.used_second_chance = false; + result.consensus_hash_first_attempt = NULL_HASH; + result.consensus_hash_second_attempt = NULL_HASH; if (available_peers.empty()) { @@ -1839,7 +1841,20 @@ namespace cn { } // Helper function to sample peers and check consensus - auto attempt_consensus = [&](const std::string& attempt_name) -> uint32_t { + // Returns: agreements count, and tracks if all responses were NULL_HASH + uint32_t total_null_hash_responses = 0; + uint32_t total_mismatches = 0; + + // Structure to return both agreement count and consensus hash + struct AttemptResult { + uint32_t agreements; + crypto::Hash consensus_hash; // Hash that M/K peers agreed on (if different from local, NULL_HASH otherwise) + }; + + auto attempt_consensus = [&](const std::string& attempt_name) -> AttemptResult { + AttemptResult attempt_result; + attempt_result.agreements = 0; + attempt_result.consensus_hash = NULL_HASH; // Randomly sample M peers from available peers, ensuring network diversity std::vector sampled_peers; std::map network_votes; // network_16 -> vote_count (capped at 1) @@ -1898,7 +1913,12 @@ namespace cn { } // Check consensus: need M agreements from K sampled peers + // Track hash votes to find consensus hash (hash with M+ votes, if different from local) + std::map hash_votes; // hash -> vote count uint32_t agreements = 0; + uint32_t null_hash_responses = 0; + uint32_t mismatches = 0; + for (uint64_t peer_id : sampled_peers) { crypto::Hash peer_hash = getPeerChunkHashFunc(peer_id); @@ -1907,12 +1927,18 @@ namespace cn { { // Peer didn't respond, timed out, or doesn't have this chunk in memory // This is not necessarily a failure - the peer might not have created this chunk yet + // or might be using version 1 (doesn't support chunk-based checkpoints) + null_hash_responses++; logger(INFO) << "Peer " << peer_id << " returned NULL_HASH for chunk " << chunk_index - << " (" << attempt_name << ") - peer may not have this chunk in memory yet"; + << " (" << attempt_name << ") - peer may not have this chunk in memory yet " + << "or may be using version 1 (doesn't support chunk checkpoints)"; continue; // Don't count as agreement or disagreement - peer doesn't have the chunk } + // Count votes for this hash + hash_votes[peer_hash]++; + if (peer_hash == local_chunk_hash) { agreements++; @@ -1921,6 +1947,7 @@ namespace cn { } else { + mismatches++; logger(WARNING) << "Peer " << peer_id << " chunk hash mismatch for chunk " << chunk_index << " (" << attempt_name << "): local=" << local_chunk_hash @@ -1928,6 +1955,27 @@ namespace cn { } } + // Find the consensus hash (hash with most votes, if >= M and different from local) + crypto::Hash consensus_hash = NULL_HASH; + uint32_t max_votes = 0; + for (const auto& vote : hash_votes) + { + if (vote.second >= req.min_agreements && vote.second > max_votes) + { + max_votes = vote.second; + consensus_hash = vote.first; + } + } + + // Only set consensus_hash if it's different from local and we have M+ agreements + if (consensus_hash != NULL_HASH && consensus_hash != local_chunk_hash && max_votes >= req.min_agreements) + { + attempt_result.consensus_hash = consensus_hash; + logger(WARNING) << "Chunk " << chunk_index << " (" << attempt_name + << "): M/K peers (" << max_votes << ") agree on different hash: " + << consensus_hash << " (local: " << local_chunk_hash << ")"; + } + // Verify we have at least n diverse networks uint32_t diverse_networks = static_cast(network_votes.size()); if (diverse_networks < req.min_diverse_networks) @@ -1936,7 +1984,9 @@ namespace cn { << " consensus " << attempt_name << ": insufficient network diversity " << "(have " << diverse_networks << " networks, need n=" << req.min_diverse_networks << ")"; // Return 0 agreements if diversity requirement not met (consensus fails) - return 0; + attempt_result.agreements = 0; + attempt_result.consensus_hash = NULL_HASH; + return attempt_result; } logger(INFO) << "Chunk " << chunk_index @@ -1945,15 +1995,22 @@ namespace cn { << req.min_agreements << " from K=" << req.min_peers << ", have n=" << diverse_networks << " diverse networks)"; - return agreements; + // Track statistics for better error messages + total_null_hash_responses += null_hash_responses; + total_mismatches += mismatches; + + attempt_result.agreements = agreements; + return attempt_result; }; // First attempt - result.agreements_first_attempt = attempt_consensus("first attempt"); + AttemptResult first_attempt = attempt_consensus("first attempt"); + result.agreements_first_attempt = first_attempt.agreements; + result.consensus_hash_first_attempt = first_attempt.consensus_hash; if (result.agreements_first_attempt >= req.min_agreements) { - // Consensus reached on first attempt + // Consensus reached on first attempt (peers agree with local hash) result.consensus_reached = true; logger(INFO) << "Chunk " << chunk_index << " consensus reached on first attempt (" @@ -1962,17 +2019,32 @@ namespace cn { } // First attempt failed - use second chance - logger(WARNING) << "Chunk " << chunk_index - << " consensus failed on first attempt (" - << result.agreements_first_attempt << " agreements, need " - << req.min_agreements << "). Re-sampling peers for second chance..."; + if (result.consensus_hash_first_attempt != NULL_HASH) + { + logger(WARNING) << "Chunk " << chunk_index + << " consensus failed on first attempt (" + << result.agreements_first_attempt << " agreements, need " + << req.min_agreements << "). " + << "M/K peers agree on different hash: " << result.consensus_hash_first_attempt + << " (local: " << local_chunk_hash << "). " + << "Re-sampling peers for second chance..."; + } + else + { + logger(WARNING) << "Chunk " << chunk_index + << " consensus failed on first attempt (" + << result.agreements_first_attempt << " agreements, need " + << req.min_agreements << "). Re-sampling peers for second chance..."; + } result.used_second_chance = true; - result.agreements_second_attempt = attempt_consensus("second attempt"); + AttemptResult second_attempt = attempt_consensus("second attempt"); + result.agreements_second_attempt = second_attempt.agreements; + result.consensus_hash_second_attempt = second_attempt.consensus_hash; if (result.agreements_second_attempt >= req.min_agreements) { - // Consensus reached on second attempt + // Consensus reached on second attempt (peers agree with local hash) result.consensus_reached = true; logger(INFO) << "Chunk " << chunk_index << " consensus reached on second attempt (" @@ -1981,11 +2053,48 @@ namespace cn { else { // Consensus failed even after second chance - logger(ERROR) << "Chunk " << chunk_index + // Check if we got the same consensus hash in both attempts (different from local) + // This indicates our blockchain diverged and we should rollback + if (result.consensus_hash_first_attempt != NULL_HASH && + result.consensus_hash_second_attempt != NULL_HASH && + result.consensus_hash_first_attempt == result.consensus_hash_second_attempt && + result.consensus_hash_first_attempt != local_chunk_hash) + { + // Same consensus hash in both attempts, different from local - ROLLBACK REQUIRED + logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index + << " ROLLBACK REQUIRED: M/K peers agree on different hash in BOTH attempts. " + << "Consensus hash: " << result.consensus_hash_first_attempt + << " (local: " << local_chunk_hash << "). " + << "This indicates blockchain divergence."; + } + else if (total_mismatches == 0 && total_null_hash_responses > 0) + { + // All peers returned NULL_HASH - this is expected if peers are v1 or don't have the chunk + logger(INFO) << "Chunk " << chunk_index + << " consensus not reached (all " << total_null_hash_responses + << " peer(s) returned NULL_HASH). This is normal if: " + << "(1) peers are using version 1 (don't support chunk checkpoints), or " + << "(2) peers haven't created this chunk yet. " + << "Chunk will remain in memory and validation will be retried when v2+ peers become available."; + } + else if (total_mismatches > 0) + { + // We got hash mismatches but not consistent consensus across both attempts + logger(WARNING) << "Chunk " << chunk_index << " consensus failed after second attempt (first: " << result.agreements_first_attempt << ", second: " << result.agreements_second_attempt << ", need " - << req.min_agreements << "). Local blockchain may be wrong."; + << req.min_agreements << "). " + << total_mismatches << " peer(s) returned different hash(es). " + << "Inconsistent results between attempts - monitoring for more mismatches."; + } + else + { + // No responses at all (shouldn't happen, but handle it) + logger(INFO) << "Chunk " << chunk_index + << " consensus not reached (no peer responses). " + << "Chunk will remain in memory and validation will be retried."; + } } return result; diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 26e4159ea..c6932aa49 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -1406,7 +1406,8 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe // Wait for response (with timeout) // In a real implementation, this should use async/await or a callback mechanism // For now, we'll use a simple polling approach with timeout - const int max_wait_ms = 5000; // 5 second timeout + // Increased timeout to handle network latency (responses were arriving 10+ seconds after request) + const int max_wait_ms = 15000; // 15 second timeout (was 5 seconds, increased for slow networks) const int poll_interval_ms = 50; int waited_ms = 0; @@ -1745,18 +1746,36 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() } } - // Trigger blockchain rollback via core - // Note: This is a critical operation - the blockchain will be rolled back to the chunk boundary - logger(ERROR, BRIGHT_RED) << "Triggering blockchain rollback to height " << rollback_height - << " due to chunk validation failure"; + // Check if rollback is required (same consensus hash in both attempts, different from local) + bool should_rollback = (result.consensus_hash_first_attempt != NULL_HASH && + result.consensus_hash_second_attempt != NULL_HASH && + result.consensus_hash_first_attempt == result.consensus_hash_second_attempt && + result.consensus_hash_first_attempt != local_chunk_hash); - // The rollback will be handled by the core's rollback mechanism - // For now, we signal the need for rollback by truncating checkpoint.dat - // The actual blockchain rollback should be triggered separately (e.g., via a flag or callback) - // TODO: Implement proper rollback trigger mechanism + if (should_rollback) + { + // Trigger blockchain rollback via existing core mechanism + // NOTE: ICore interface doesn't expose rollback_chain_to, but Core class does + // For now, we log the rollback requirement. The actual rollback should be triggered + // via a callback or by adding rollback_chain_to to ICore interface + logger(ERROR, BRIGHT_RED) << "ROLLBACK REQUIRED: Chunk " << chunk_index + << " validation failed - same consensus hash in both attempts. " + << "Consensus hash: " << result.consensus_hash_first_attempt + << " (local: " << local_chunk_hash << "). " + << "Rollback to height " << rollback_height + << " is required. " + << "TODO: Implement rollback trigger mechanism (ICore interface needs rollback_chain_to method)"; + } + else + { + // Consensus failed but no rollback required (inconsistent results, NULL_HASH responses, etc.) + logger(WARNING) << "Chunk " << chunk_index + << " consensus failed but rollback not required (inconsistent results between attempts). " + << "Chunk will remain in memory and validation will be retried."; + } - // Stop validation - we found the divergence point - // The node will need to resync from the rollback height + // Stop validation - we found the divergence point (or need to wait for peers) + // The node will need to resync from the rollback height (if rollback occurred) break; } } From 39217115535fc0a1a802f1293257fd7f975ed693 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 08:49:14 -0500 Subject: [PATCH 10/56] fix to unordered_map --- src/CryptoNoteCore/CheckpointsList.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 4b402e7fd..e4ea1d793 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -71,9 +71,11 @@ #include #include #include +#include #include #include #include +#include #include "CheckpointList.h" @@ -1914,7 +1916,8 @@ namespace cn { // Check consensus: need M agreements from K sampled peers // Track hash votes to find consensus hash (hash with M+ votes, if different from local) - std::map hash_votes; // hash -> vote count + // Use unordered_map since crypto::Hash doesn't have comparison operator for std::map + std::unordered_map> hash_votes; // hash -> vote count uint32_t agreements = 0; uint32_t null_hash_responses = 0; uint32_t mismatches = 0; From 1703660c0f0c5eb2259bf83ffee020dc00ce1c69 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 09:21:55 -0500 Subject: [PATCH 11/56] increased timeout for chunk hash response --- .../CryptoNoteProtocolHandler.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index c6932aa49..a1b7d704d 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -1406,8 +1406,9 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe // Wait for response (with timeout) // In a real implementation, this should use async/await or a callback mechanism // For now, we'll use a simple polling approach with timeout - // Increased timeout to handle network latency (responses were arriving 10+ seconds after request) - const int max_wait_ms = 15000; // 15 second timeout (was 5 seconds, increased for slow networks) + // Increased timeout to handle network latency (responses were arriving 30+ seconds after request) + // 45 seconds should be sufficient for slow networks, but this is unusually high latency + const int max_wait_ms = 45000; // 45 second timeout (was 15 seconds, increased for very slow networks) const int poll_interval_ms = 50; int waited_ms = 0; @@ -1426,6 +1427,19 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe } } + // Timeout reached - check one more time for late response (network delays can cause responses to arrive just after timeout) + { + std::lock_guard lock(m_pending_chunk_hashes_mutex); + auto it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); + if (it != m_pending_chunk_hashes.end()) { + crypto::Hash result = it->second; + m_pending_chunk_hashes.erase(it); + logger(INFO) << "Received late chunk hash response from peer " << peer_id << " " << *peer_context + << " for chunk " << chunk_index << " (arrived after timeout): " << result; + return result; + } + } + logger(logging::WARNING) << "Timeout waiting for chunk hash response from peer " << peer_id << " " << *peer_context << " for chunk " << chunk_index; return NULL_HASH; From 7537525bbf2f0708f8ea38ec4aea014a3ef30fff Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 09:39:02 -0500 Subject: [PATCH 12/56] handle late chunk hash responses --- .../CryptoNoteProtocolHandler.cpp | 70 ++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index a1b7d704d..6751cfa17 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -1332,11 +1332,37 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE logger(INFO) << context << " Received chunk hash response from peer " << peer_id << " for chunk " << arg.chunk_index; + // Check if we're still validating this chunk (late responses might arrive after timeout) + bool still_validating = false; + { + std::lock_guard lock(m_chunk_validation_mutex); + still_validating = (m_current_validating_chunk_index == arg.chunk_index); + } + // Store the response in pending chunk hashes map for the consensus mechanism // NOTE: NULL_HASH is a valid response (means peer doesn't have this chunk) + bool was_pending = false; { std::lock_guard lock(m_pending_chunk_hashes_mutex); - m_pending_chunk_hashes[std::make_pair(peer_id, arg.chunk_index)] = arg.chunk_hash; + auto key = std::make_pair(peer_id, arg.chunk_index); + was_pending = (m_pending_chunk_hashes.find(key) != m_pending_chunk_hashes.end()); + m_pending_chunk_hashes[key] = arg.chunk_hash; + } + + // Log if this is a late response (arrived after timeout) + if (!still_validating && was_pending) + { + logger(INFO) << context << " Received LATE chunk hash response from peer " << peer_id + << " for chunk " << arg.chunk_index + << " (validation already completed - response arrived after 5s timeout). " + << "Response cached and will be used in next validation attempt."; + } + else if (!still_validating && !was_pending) + { + // Response arrived for a request we already gave up on - this is fine, we'll use it next time + logger(DEBUGGING) << context << " Received chunk hash response from peer " << peer_id + << " for chunk " << arg.chunk_index + << " (from previous request, will be used in next validation attempt)"; } if (arg.chunk_hash == NULL_HASH) @@ -1377,10 +1403,18 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe return NULL_HASH; } - // Clear any previous pending response for this peer/chunk + // Check if we already have a cached response from a previous request (late response) + // This allows us to use responses that arrived after the timeout { std::lock_guard lock(m_pending_chunk_hashes_mutex); - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + auto it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); + if (it != m_pending_chunk_hashes.end()) { + crypto::Hash cached_result = it->second; + m_pending_chunk_hashes.erase(it); + logger(INFO) << "Using cached chunk hash response from peer " << peer_id << " " << *peer_context + << " for chunk " << chunk_index << " (from previous request): " << cached_result; + return cached_result; + } } // Send request @@ -1403,12 +1437,10 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe logger(INFO) << "Successfully sent chunk hash request for chunk " << chunk_index << " to peer " << peer_id << " " << *peer_context << " (waiting for response...)"; - // Wait for response (with timeout) - // In a real implementation, this should use async/await or a callback mechanism - // For now, we'll use a simple polling approach with timeout - // Increased timeout to handle network latency (responses were arriving 30+ seconds after request) - // 45 seconds should be sufficient for slow networks, but this is unusually high latency - const int max_wait_ms = 45000; // 45 second timeout (was 15 seconds, increased for very slow networks) + // Wait for response (with short timeout) + // Use a short timeout (5 seconds) to avoid blocking the P2P message queue + // If response arrives later, it will be stored in m_pending_chunk_hashes and used in next validation attempt + const int max_wait_ms = 5000; // 5 second timeout - responses arriving later will be used in next attempt const int poll_interval_ms = 50; int waited_ms = 0; @@ -1427,21 +1459,11 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe } } - // Timeout reached - check one more time for late response (network delays can cause responses to arrive just after timeout) - { - std::lock_guard lock(m_pending_chunk_hashes_mutex); - auto it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); - if (it != m_pending_chunk_hashes.end()) { - crypto::Hash result = it->second; - m_pending_chunk_hashes.erase(it); - logger(INFO) << "Received late chunk hash response from peer " << peer_id << " " << *peer_context - << " for chunk " << chunk_index << " (arrived after timeout): " << result; - return result; - } - } - - logger(logging::WARNING) << "Timeout waiting for chunk hash response from peer " << peer_id - << " " << *peer_context << " for chunk " << chunk_index; + // Timeout reached - response may arrive later and will be used in next validation attempt + // Don't remove the pending entry - keep it so late responses can be used + logger(DEBUGGING) << "Timeout waiting for immediate chunk hash response from peer " << peer_id + << " " << *peer_context << " for chunk " << chunk_index + << " (response may arrive later and will be used in next validation attempt)"; return NULL_HASH; } From ccf7f8fb44c75a73238d6ea60b3d223614afda66 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 10:30:17 -0500 Subject: [PATCH 13/56] p2p chunk sequence --- .../CryptoNoteProtocolHandler.cpp | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 6751cfa17..cce0ebb44 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -1407,7 +1407,8 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe // This allows us to use responses that arrived after the timeout { std::lock_guard lock(m_pending_chunk_hashes_mutex); - auto it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); + auto key = std::make_pair(peer_id, chunk_index); + auto it = m_pending_chunk_hashes.find(key); if (it != m_pending_chunk_hashes.end()) { crypto::Hash cached_result = it->second; m_pending_chunk_hashes.erase(it); @@ -1415,6 +1416,22 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe << " for chunk " << chunk_index << " (from previous request): " << cached_result; return cached_result; } + // Log for debugging - show what pending entries exist + if (!m_pending_chunk_hashes.empty()) { + logger(DEBUGGING) << "No cached response for peer " << peer_id << " chunk " << chunk_index + << ". Total pending entries: " << m_pending_chunk_hashes.size(); + // Log first few pending entries to see what we have + int count = 0; + for (const auto& entry : m_pending_chunk_hashes) { + if (count++ < 3) { + logger(DEBUGGING) << " Pending: peer " << entry.first.first << " chunk " << entry.first.second + << " hash " << entry.second; + } + } + } else { + logger(DEBUGGING) << "No cached response for peer " << peer_id << " chunk " << chunk_index + << " (no pending entries)"; + } } // Send request @@ -1431,6 +1448,11 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe if (!sent) { logger(WARNING) << "Failed to send chunk hash request to peer " << peer_id << " " << *peer_context << " for chunk " << chunk_index; + // Remove the pending entry we just created + { + std::lock_guard lock(m_pending_chunk_hashes_mutex); + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } return NULL_HASH; } @@ -1460,7 +1482,7 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe } // Timeout reached - response may arrive later and will be used in next validation attempt - // Don't remove the pending entry - keep it so late responses can be used + // The response handler will store it in m_pending_chunk_hashes when it arrives logger(DEBUGGING) << "Timeout waiting for immediate chunk hash response from peer " << peer_id << " " << *peer_context << " for chunk " << chunk_index << " (response may arrive later and will be used in next validation attempt)"; From 621ee51b632289eddc52cb6472a99dbfd059ba45 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 13:26:29 -0500 Subject: [PATCH 14/56] async chunk validation --- .../CryptoNoteProtocolHandler.cpp | 451 +++++++++++++----- .../CryptoNoteProtocolHandler.h | 32 +- 2 files changed, 368 insertions(+), 115 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index cce0ebb44..bc7f5d2c0 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -13,8 +13,12 @@ #include #include #include +#include +#include #include #include +#include +#include #include #include #include @@ -674,6 +678,9 @@ bool CryptoNoteProtocolHandler::on_idle() // Only for version 2+ nodes (chunked checkpoint system) if (cn::P2P_CURRENT_VERSION >= cn::P2P_CHECKPOINT_LIST_VERSION) { + // Check pending validations for consensus (asynchronous approach) + check_pending_chunk_validations(); + // Start new validations if needed validate_unverified_chunks(); } @@ -1341,12 +1348,16 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE // Store the response in pending chunk hashes map for the consensus mechanism // NOTE: NULL_HASH is a valid response (means peer doesn't have this chunk) + uint64_t response_time = time(nullptr); bool was_pending = false; { std::lock_guard lock(m_pending_chunk_hashes_mutex); auto key = std::make_pair(peer_id, arg.chunk_index); was_pending = (m_pending_chunk_hashes.find(key) != m_pending_chunk_hashes.end()); - m_pending_chunk_hashes[key] = arg.chunk_hash; + ChunkHashResponse response; + response.hash = arg.chunk_hash; + response.timestamp = response_time; + m_pending_chunk_hashes[key] = response; } // Log if this is a late response (arrived after timeout) @@ -1378,6 +1389,40 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE return 1; } +bool CryptoNoteProtocolHandler::send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index) +{ + // Find the peer connection + CryptoNoteConnectionContext* peer_context = nullptr; + m_p2p->for_each_connection([&peer_context, peer_id](CryptoNoteConnectionContext& ctx, uint64_t id) { + if (id == peer_id) { + peer_context = &ctx; + } + }); + + if (!peer_context) { + logger(WARNING) << "Cannot send async chunk hash request to peer " << peer_id << ": peer not found"; + return false; + } + + // Send request (non-blocking) + NOTIFY_REQUEST_CHUNK_HASH::request req; + req.chunk_index = chunk_index; + + logger(INFO) << "Sending async chunk hash request for chunk " << chunk_index << " to peer " << peer_id + << " " << *peer_context; + + bool sent = post_notify(*m_p2p, req, *peer_context); + if (!sent) { + logger(WARNING) << "Failed to send async chunk hash request to peer " << peer_id << " " << *peer_context + << " for chunk " << chunk_index; + return false; + } + + logger(INFO) << "Successfully sent async chunk hash request for chunk " << chunk_index + << " to peer " << peer_id << " " << *peer_context; + return true; +} + crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t peer_id, uint32_t chunk_index) { // Find the peer connection @@ -1410,7 +1455,7 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe auto key = std::make_pair(peer_id, chunk_index); auto it = m_pending_chunk_hashes.find(key); if (it != m_pending_chunk_hashes.end()) { - crypto::Hash cached_result = it->second; + crypto::Hash cached_result = it->second.hash; m_pending_chunk_hashes.erase(it); logger(INFO) << "Using cached chunk hash response from peer " << peer_id << " " << *peer_context << " for chunk " << chunk_index << " (from previous request): " << cached_result; @@ -1425,7 +1470,7 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe for (const auto& entry : m_pending_chunk_hashes) { if (count++ < 3) { logger(DEBUGGING) << " Pending: peer " << entry.first.first << " chunk " << entry.first.second - << " hash " << entry.second; + << " hash " << entry.second.hash << " (received at " << entry.second.timestamp << ")"; } } } else { @@ -1473,7 +1518,7 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe std::lock_guard lock(m_pending_chunk_hashes_mutex); auto it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); if (it != m_pending_chunk_hashes.end()) { - crypto::Hash result = it->second; + crypto::Hash result = it->second.hash; m_pending_chunk_hashes.erase(it); logger(INFO) << "Received chunk hash response from peer " << peer_id << " " << *peer_context << " for chunk " << chunk_index << ": " << result; @@ -1685,160 +1730,342 @@ bool CryptoNoteProtocolHandler::validate_unverified_chunks() logger(INFO) << "Validating chunk " << chunk_index << " (oldest unverified chunk) " << "with " << eligible_peers.size() << " eligible peer(s)"; - // Prepare functions for consensus mechanism - uint32_t validating_chunk = chunk_index; // Capture chunk_index for lambda - auto getPeerChunkHashFunc = [this, validating_chunk](uint64_t peer_id) -> crypto::Hash { - // Request chunk hash from peer (with timeout) - crypto::Hash peer_hash = request_chunk_hash_from_peer(peer_id, validating_chunk); - if (peer_hash == NULL_HASH) + // Check if this chunk is already being validated asynchronously + { + std::lock_guard lock(m_pending_validations_mutex); + if (m_pending_validations.find(chunk_index) != m_pending_validations.end()) + { + logger(DEBUGGING) << "Chunk " << chunk_index << " is already being validated asynchronously, skipping"; + { + std::lock_guard lock2(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + } + + // Calculate consensus requirements (M, K, n) + CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(eligible_peers.size()); + logger(INFO) << "Consensus requirements (testnet): M=" << req.min_agreements + << ", K=" << req.min_peers << ", n=" << req.min_diverse_networks + << " (have " << eligible_peers.size() << " available peers)"; + + // Sample K peers with network diversity (same logic as verify_chunk_with_peer_consensus) + std::vector sampled_peers; + std::map network_votes; // network_16 -> vote_count (capped at 1) + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, eligible_peers.size() - 1); + + // Sample K peers (req.min_peers) with network diversity + while (sampled_peers.size() < req.min_peers && sampled_peers.size() < eligible_peers.size()) + { + size_t attempts = 0; + const size_t max_attempts = eligible_peers.size() * 2; + + while (attempts < max_attempts && sampled_peers.size() < req.min_peers) + { + size_t idx = dis(gen); + uint64_t peer_id = eligible_peers[idx]; + + // Check if we already sampled this peer + if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) != sampled_peers.end()) + { + attempts++; + continue; + } + + // Check network diversity (max 1 vote per /16 network) + uint32_t net16 = peer_network_16[peer_id]; + if (network_votes[net16] >= 1 && sampled_peers.size() < eligible_peers.size()) + { + attempts++; + continue; + } + + // Accept this peer + sampled_peers.push_back(peer_id); + network_votes[net16] = std::min(network_votes[net16] + 1, 1U); + break; + } + + // If we couldn't find enough diverse peers, relax diversity requirement + if (sampled_peers.size() < req.min_peers && attempts >= max_attempts) { - // Find peer context to log IP address - std::string peer_info = "peer " + std::to_string(peer_id); - m_p2p->for_each_connection([&peer_info, peer_id](const CryptoNoteConnectionContext& ctx, uint64_t id) { - if (id == peer_id) { - std::ostringstream oss; - oss << ctx; - peer_info = "peer " + std::to_string(peer_id) + " " + oss.str(); + for (uint64_t peer_id : eligible_peers) + { + if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) == sampled_peers.end()) + { + sampled_peers.push_back(peer_id); + if (sampled_peers.size() >= req.min_peers) + break; } - }); - logger(INFO) << peer_info << " does not have chunk " << validating_chunk - << " in memory (returned NULL_HASH)"; + } + } + } + + if (sampled_peers.empty()) + { + logger(WARNING) << "Could not sample any peers for chunk " << chunk_index << " validation"; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + + // Verify we have at least n distinct networks (network diversity requirement) + std::set distinct_networks; + for (uint64_t peer_id : sampled_peers) + { + distinct_networks.insert(peer_network_16[peer_id]); + } + + if (distinct_networks.size() < req.min_diverse_networks) + { + logger(WARNING) << "Could not achieve network diversity for chunk " << chunk_index + << " validation: have " << distinct_networks.size() + << " distinct networks, need " << req.min_diverse_networks + << ". Sampled " << sampled_peers.size() << " peer(s)."; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + + logger(INFO) << "Sampled " << sampled_peers.size() << " peer(s) from " << distinct_networks.size() + << " distinct network(s) (requirement: " << req.min_diverse_networks << " networks)"; + + // Send async requests to sampled peers + uint64_t request_time = time(nullptr); + uint32_t requests_sent = 0; + for (uint64_t peer_id : sampled_peers) + { + if (send_chunk_hash_request_async(peer_id, chunk_index)) + { + requests_sent++; } - return peer_hash; - }; + } - auto getPeerNetwork16Func = [&peer_network_16](uint64_t peer_id) -> uint32_t { - auto it = peer_network_16.find(peer_id); - if (it != peer_network_16.end()) + if (requests_sent == 0) + { + logger(WARNING) << "Failed to send any async requests for chunk " << chunk_index; { - return it->second; + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; } - return 0; // Default network if not found - }; + continue; + } - // Verify chunk with peer consensus - CheckpointList::ConsensusResult result = m_core.getCheckpointList().verify_chunk_with_peer_consensus( - chunk_index, - local_chunk_hash, - getPeerChunkHashFunc, - eligible_peers, - getPeerNetwork16Func - ); + // Store pending validation state + { + std::lock_guard lock(m_pending_validations_mutex); + PendingChunkValidation pending; + pending.chunk_index = chunk_index; + pending.request_timestamp = request_time; + pending.attempt_start_time = request_time; + pending.attempt_number = 1; + pending.requested_peers = sampled_peers; + pending.local_hash = local_chunk_hash; + pending.is_first_attempt = true; + m_pending_validations[chunk_index] = pending; + } + + logger(INFO) << "Sent async chunk hash requests for chunk " << chunk_index + << " to " << requests_sent << " peer(s). " + << "Will check for consensus after 2 minutes."; - // Clear validation state + // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) { std::lock_guard lock(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; } - // Check if all peers returned NULL_HASH (they don't have this chunk yet) - // In this case, we should wait for peers to create the chunk, not fail consensus - if (!result.consensus_reached && result.agreements_first_attempt == 0 && result.agreements_second_attempt == 0) + // Don't process next chunk until this one is validated (file must be sequential) + // The check_pending_chunk_validations() function will handle consensus checking + break; + } + + return true; +} + +void CryptoNoteProtocolHandler::check_pending_chunk_validations() +{ + uint64_t time_now = time(nullptr); + const uint64_t CONSENSUS_WAIT_SECONDS = 120; // 2 minutes + const uint64_t RETRY_DELAY_SECONDS = 60; // 1 minute delay before retry + + std::lock_guard lock(m_pending_validations_mutex); + + // Iterate through pending validations + for (auto it = m_pending_validations.begin(); it != m_pending_validations.end();) + { + uint32_t chunk_index = it->first; + PendingChunkValidation& pending = it->second; + + uint64_t elapsed = time_now - pending.request_timestamp; + + // Check if 2 minutes have passed since requests were sent + if (elapsed < CONSENSUS_WAIT_SECONDS) + { + // Not enough time has passed yet, skip this validation + ++it; + continue; + } + + // 2 minutes have passed - check for consensus + logger(INFO) << "Checking consensus for chunk " << chunk_index + << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; + + // Collect responses from requested peers + std::unordered_map> hash_votes; // hash -> vote count + uint32_t agreements = 0; + uint32_t null_hash_responses = 0; + uint32_t responses_received = 0; + { - logger(INFO) << "Chunk " << chunk_index - << " validation: No peers have this chunk in memory yet. " - << "Will retry validation once peers create this chunk. " - << "Stopping validation - no point to add subsequent chunks to checkpoint.dat " - << "if chunk " << chunk_index << " is not validated first (file must be sequential)."; - // Don't rollback - peers just need to create the chunk first - break; // Stop - file must be sequential, no gaps allowed + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + + for (uint64_t peer_id : pending.requested_peers) + { + auto response_it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); + if (response_it != m_pending_chunk_hashes.end()) + { + crypto::Hash peer_hash = response_it->second.hash; + responses_received++; + + if (peer_hash == NULL_HASH) + { + null_hash_responses++; + continue; + } + + // Count votes for this hash + hash_votes[peer_hash]++; + + if (peer_hash == pending.local_hash) + { + agreements++; + } + } + } } - if (result.consensus_reached) + // Calculate consensus requirements + CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(pending.requested_peers.size()); + + logger(INFO) << "Chunk " << chunk_index << " consensus check: received " << responses_received + << " response(s) from " << pending.requested_peers.size() << " requested peer(s). " + << "Agreements: " << agreements << " (need M=" << req.min_agreements << "), " + << "NULL_HASH responses: " << null_hash_responses; + + // Check if we have M agreements (consensus reached) + if (agreements >= req.min_agreements) { - // Consensus reached - add chunk to checkpoint.dat + // Consensus reached - save to checkpoint.dat + logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index + << " validated via peer consensus (" << agreements + << " agreements, need M=" << req.min_agreements << ")"; + if (m_core.getCheckpointList().add_verified_chunk_to_file(chunk_index)) { logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index - << " validated and saved to checkpoint.dat via peer consensus"; + << " saved to checkpoint.dat"; - // Continue to next chunk + // Remove pending validation + it = m_pending_validations.erase(it); + + // Clean up responses for this chunk + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } continue; } else { logger(ERROR) << "Failed to save validated chunk " << chunk_index << " to checkpoint.dat"; - break; // Stop validation if we can't save + // Remove pending validation anyway (we'll retry later) + it = m_pending_validations.erase(it); + continue; } } - else + + // Consensus not reached - check if we should retry + if (pending.is_first_attempt) { - // Consensus failed - this indicates our blockchain diverged - // Rollback to the previous chunk boundary - logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index - << " validation FAILED: peer consensus did not agree with local chunk hash. " - << "This indicates blockchain divergence. Rolling back to chunk " - << (chunk_index > 0 ? chunk_index - 1 : 0) << " boundary."; - - // Calculate rollback height - // SIMPLIFIED: Block 0 (genesis) is NOT in any chunk - // chunk[0]: blocks 1 to chunk_size - // chunk[1]: blocks (chunk_size + 1) to (2 * chunk_size) - // chunk[N]: blocks (N * chunk_size + 1) to ((N + 1) * chunk_size) - // - // Rollback to end of previous chunk (chunk_index - 1) - uint32_t chunk_size = m_core.getCheckpointList().get_chunk_size(); - uint32_t rollback_height; - if (chunk_index == 0) + // First attempt failed - wait 1 more minute (3 minutes total) before retry + uint64_t total_elapsed = time_now - pending.attempt_start_time; + if (total_elapsed < (CONSENSUS_WAIT_SECONDS + RETRY_DELAY_SECONDS)) { - rollback_height = 0; // Rollback to genesis - } - else - { - // Rollback to end of previous chunk - // Previous chunk (chunk_index - 1) ends at: chunk_index * chunk_size - // Example: If chunk 38 fails, rollback to: 38 * 10000 = 380,000 (end of chunk 37) - rollback_height = chunk_index * chunk_size; + // Still waiting for retry delay + ++it; + continue; } - // Trigger blockchain rollback - logger(ERROR, BRIGHT_RED) << "Rolling back blockchain to height " << rollback_height - << " (chunk " << (chunk_index > 0 ? chunk_index - 1 : 0) << " boundary)"; + // Retry delay passed - start second attempt + logger(INFO) << "Chunk " << chunk_index + << " consensus failed on first attempt. Starting second attempt..."; - // Truncate checkpoint.dat to previous chunk - if (chunk_index > 0) + // Send new requests to same peers (or get new eligible peers) + // For now, reuse same peers + uint64_t retry_time = time_now; + uint32_t requests_sent = 0; + for (uint64_t peer_id : pending.requested_peers) { - if (!m_core.getCheckpointList().truncate_checkpoint_file(chunk_index - 1)) + if (send_chunk_hash_request_async(peer_id, chunk_index)) { - logger(ERROR) << "Failed to truncate checkpoint.dat after validation failure"; + requests_sent++; } } - // Check if rollback is required (same consensus hash in both attempts, different from local) - bool should_rollback = (result.consensus_hash_first_attempt != NULL_HASH && - result.consensus_hash_second_attempt != NULL_HASH && - result.consensus_hash_first_attempt == result.consensus_hash_second_attempt && - result.consensus_hash_first_attempt != local_chunk_hash); - - if (should_rollback) + if (requests_sent > 0) { - // Trigger blockchain rollback via existing core mechanism - // NOTE: ICore interface doesn't expose rollback_chain_to, but Core class does - // For now, we log the rollback requirement. The actual rollback should be triggered - // via a callback or by adding rollback_chain_to to ICore interface - logger(ERROR, BRIGHT_RED) << "ROLLBACK REQUIRED: Chunk " << chunk_index - << " validation failed - same consensus hash in both attempts. " - << "Consensus hash: " << result.consensus_hash_first_attempt - << " (local: " << local_chunk_hash << "). " - << "Rollback to height " << rollback_height - << " is required. " - << "TODO: Implement rollback trigger mechanism (ICore interface needs rollback_chain_to method)"; + // Update pending validation for second attempt + pending.request_timestamp = retry_time; + pending.attempt_number = 2; + pending.is_first_attempt = false; + + logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index + << " to " << requests_sent << " peer(s). " + << "Will check for consensus after 2 minutes."; } else { - // Consensus failed but no rollback required (inconsistent results, NULL_HASH responses, etc.) - logger(WARNING) << "Chunk " << chunk_index - << " consensus failed but rollback not required (inconsistent results between attempts). " - << "Chunk will remain in memory and validation will be retried."; + logger(WARNING) << "Failed to send second attempt requests for chunk " << chunk_index; + // Remove pending validation + it = m_pending_validations.erase(it); + continue; } + } + else + { + // Second attempt also failed - consensus failed + logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index + << " validation FAILED: peer consensus did not agree with local chunk hash " + << "after 2 attempts. This indicates blockchain divergence."; - // Stop validation - we found the divergence point (or need to wait for peers) - // The node will need to resync from the rollback height (if rollback occurred) - break; + // TODO: Handle rollback (same logic as before) + // For now, just remove pending validation + it = m_pending_validations.erase(it); + + // Clean up responses + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } + continue; } + + ++it; } - - return true; } }; // namespace cn diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h index e6ac8487b..d4f57dfd7 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h @@ -134,10 +134,27 @@ namespace cn std::atomic m_maxObjectCount; - // Pending chunk hash requests: (peer_id, chunk_index) -> chunk_hash - // Used for consensus mechanism to request chunk hashes from peers + // Pending chunk hash responses: (peer_id, chunk_index) -> (hash, timestamp) + // Used for asynchronous consensus mechanism + struct ChunkHashResponse { + crypto::Hash hash; + uint64_t timestamp; // When response was received + }; mutable std::mutex m_pending_chunk_hashes_mutex; - std::map, crypto::Hash> m_pending_chunk_hashes; + std::map, ChunkHashResponse> m_pending_chunk_hashes; + + // Pending chunk validation attempts: chunk_index -> validation state + struct PendingChunkValidation { + uint32_t chunk_index; + uint64_t request_timestamp; // When requests were sent + uint64_t attempt_start_time; // When this attempt started + uint32_t attempt_number; // 1 or 2 + std::vector requested_peers; // Peers we requested from + crypto::Hash local_hash; + bool is_first_attempt; + }; + mutable std::mutex m_pending_validations_mutex; + std::map m_pending_validations; // chunk_index -> validation state // Chunk validation state: track which chunk we're currently validating // This prevents duplicate validation attempts and ensures chronological order @@ -150,5 +167,14 @@ namespace cn // Only validates when peers meet uptime requirements // @return true if validation was attempted (even if it failed) bool validate_unverified_chunks(); + + // Check pending chunk validations for consensus (called periodically) + // Checks if we have M identical hashes within the time window (2 minutes) + // If consensus reached, processes it; if timeout, schedules retry + void check_pending_chunk_validations(); + + // Send chunk hash request asynchronously (non-blocking) + // Returns true if request was sent successfully + bool send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index); }; } From 434256e3aa605b28fd3f51a95957cedbd08bb383 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 16:23:09 -0500 Subject: [PATCH 15/56] simplify code --- src/CryptoNoteCore/CheckpointList.h | 119 +++ src/CryptoNoteCore/CheckpointsList.cpp | 912 ++++++++++-------- .../CryptoNoteProtocolHandler.cpp | 679 +------------ .../CryptoNoteProtocolHandler.h | 47 +- .../CryptoNoteProtocolHandlerChunk.cpp | 691 +++++++++++++ .../CryptoNoteProtocolHandlerChunk.h | 105 ++ 6 files changed, 1445 insertions(+), 1108 deletions(-) create mode 100644 src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp create mode 100644 src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index 5f5bac8b0..aec291327 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -129,6 +129,44 @@ namespace cn // } static uint32_t get_network_16(uint32_t ip); + // Structure to return peer sampling results + struct PeerSamplingResult { + std::vector sampled_peers; + std::map network_votes; // network_16 -> vote_count (capped at 1) + }; + + /** + * Sample peers with network diversity + * + * Randomly samples K peers from available peers, ensuring network diversity + * (max 1 peer per /16 network). Falls back to any available peer if diversity + * cannot be achieved. + * + * @param available_peers List of available peer IDs + * @param getPeerNetwork16Func Function to get /16 network for a peer (peer_id -> network_16) + * @param num_peers_to_sample Number of peers to sample (K) + * @return PeerSamplingResult with sampled peers and network votes + */ + static PeerSamplingResult sample_peers_with_diversity( + const std::vector& available_peers, + std::function getPeerNetwork16Func, + size_t num_peers_to_sample); + + /** + * Fallback peer selection when diversity requirement cannot be met + * + * When we can't find enough diverse peers, relax the diversity requirement + * and select any available peers to reach the target sample size. + * + * @param available_peers List of all available peer IDs + * @param sampled_peers Already sampled peers (will be updated) + * @param target_size Target number of peers to sample + */ + static void fallback_peer_selection( + const std::vector& available_peers, + std::vector& sampled_peers, + size_t target_size); + // Health metrics for monitoring struct HealthMetrics { uint32_t total_chunks; @@ -504,5 +542,86 @@ namespace cn return m_valid_point_sizes.find(fsize) != m_valid_point_sizes.end(); } + + // Internal structure for consensus attempt results + struct AttemptResult { + uint32_t agreements; + crypto::Hash consensus_hash; // Hash that M/K peers agreed on (if different from local, NULL_HASH otherwise) + }; + + /** + * Attempt to reach consensus on a chunk hash with peers + * + * Samples K peers with network diversity, collects their chunk hashes, + * and determines if M peers agree with the local hash. + * + * @param chunk_index The chunk index being validated + * @param local_chunk_hash The locally computed chunk hash + * @param getPeerChunkHashFunc Function to get chunk hash from a peer + * @param available_peers List of available peer IDs + * @param getPeerNetwork16Func Function to get /16 network for a peer + * @param req Consensus requirements (M, K, n) + * @param attempt_name Name of the attempt (for logging) + * @param total_null_hash_responses Reference to counter for NULL_HASH responses (updated) + * @param total_mismatches Reference to counter for mismatches (updated) + * @return AttemptResult with agreement count and consensus hash + */ + AttemptResult attempt_consensus_impl( + uint32_t chunk_index, + const crypto::Hash& local_chunk_hash, + std::function getPeerChunkHashFunc, + const std::vector& available_peers, + std::function getPeerNetwork16Func, + const ConsensusRequirements& req, + const std::string& attempt_name, + uint32_t& total_null_hash_responses, + uint32_t& total_mismatches) const; + + // Internal structure for checkpoint application results + struct CheckpointApplicationResult { + bool success; + uint32_t checkpoints_from_config; + uint32_t checkpoints_from_dns; + std::vector checkpoints_in_chunk; // Heights of checkpoints applied (optional) + }; + + /** + * Apply checkpoint priority to chunk block IDs + * + * Applies checkpoints with priority order: CryptoNoteConfig.h > DNS > blockchain.dat + * Validates that blockchain.dat matches expected checkpoint values before applying. + * + * @param chunk_index The chunk index being processed + * @param chunk_start_height Starting height of the chunk + * @param chunk_end_height Ending height of the chunk + * @param chunk_block_ids Reference to chunk block IDs (modified in place) + * @param getBlockIdsFunc Optional function to re-fetch block IDs for validation (nullptr if not needed) + * @param track_checkpoint_heights If true, tracks which specific checkpoint heights were applied + * @return CheckpointApplicationResult with success status and counters + */ + CheckpointApplicationResult apply_checkpoint_priority_to_chunk( + uint32_t chunk_index, + uint32_t chunk_start_height, + uint32_t chunk_end_height, + std::vector& chunk_block_ids, + std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc = nullptr, + bool track_checkpoint_heights = false) const; + + /** + * Validate a checkpoint hash against blockchain data + * + * @param checkpoint_height The checkpoint height to validate + * @param expected_hash The expected hash from checkpoint source (CryptoNoteConfig.h or DNS) + * @param chunk_block_ids Current chunk block IDs (may have been modified by DNS checkpoints) + * @param index_in_chunk Index of checkpoint height within the chunk + * @param getBlockIdsFunc Optional function to re-fetch original blockchain data (nullptr if not needed) + * @return true if validation passed, false otherwise + */ + bool validate_checkpoint_hash( + uint32_t checkpoint_height, + const crypto::Hash& expected_hash, + const std::vector& chunk_block_ids, + uint32_t index_in_chunk, + std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc = nullptr) const; }; } diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index e4ea1d793..163a3c4bc 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -378,125 +378,33 @@ namespace cn { return false; } - // PRIORITY ORDER for block hashes in chunk (applied in reverse order to maintain priority): - // 1. Start with blockchain.dat (base data) - // 2. Apply DNS checkpoints (overwrites blockchain.dat) - // 3. Apply CryptoNoteConfig.h checkpoints (overwrites both DNS and blockchain.dat) - // Final priority: CryptoNoteConfig.h > DNS > blockchain.dat - - std::vector checkpoints_in_chunk; - uint32_t checkpoints_from_config = 0; - uint32_t checkpoints_from_dns = 0; - - // STEP 1: Apply DNS checkpoints (PRIORITY 2) - // These will be overwritten by CryptoNoteConfig.h if there's a conflict - for (const auto& checkpoint : m_dns_checkpoint_hashes) + // Apply checkpoint priority: CryptoNoteConfig.h > DNS > blockchain.dat + CheckpointApplicationResult checkpoint_result = apply_checkpoint_priority_to_chunk( + chunk_index, + chunk_start_height, + chunk_end_height, + chunk_block_ids, + getBlockIdsFunc, + true); // track_checkpoint_heights = true for detailed logging + + if (!checkpoint_result.success) { - uint32_t checkpoint_height = checkpoint.first; - if (checkpoint_height >= chunk_start_height && checkpoint_height <= chunk_end_height) - { - uint32_t index_in_chunk = checkpoint_height - chunk_start_height; - - if (index_in_chunk >= chunk_block_ids.size()) - { - logger(ERROR) << "DNS checkpoint height " << checkpoint_height - << " index out of range in chunk " << chunk_index; - m_chunks.clear(); - return false; - } - - // VALIDATION: Verify that blockchain.dat matches the DNS checkpoint - const crypto::Hash& blockchain_hash = chunk_block_ids[index_in_chunk]; - const crypto::Hash& dns_hash = checkpoint.second; - - if (blockchain_hash != dns_hash) - { - logger(ERROR) << "DNS CHECKPOINT VALIDATION FAILED for chunk " - << chunk_index << " at height " << checkpoint_height << "!" - << " Expected (from DNS): " << dns_hash - << ", Got (from blockchain.dat): " << blockchain_hash - << ". Cannot create chunk - blockchain validation failed."; - m_chunks.clear(); - return false; - } - - // Replace with DNS checkpoint hash - chunk_block_ids[index_in_chunk] = dns_hash; - checkpoints_from_dns++; - checkpoints_in_chunk.push_back(checkpoint_height); - - logger(DEBUGGING) << "Applied DNS checkpoint hash for height " - << checkpoint_height << " in chunk " << chunk_index; - } - } - - // STEP 2: Apply CryptoNoteConfig.h checkpoints (PRIORITY 1 - HIGHEST) - // These overwrite both blockchain.dat and DNS checkpoints - for (const auto& checkpoint : m_old_checkpoint_hashes) - { - uint32_t checkpoint_height = checkpoint.first; - if (checkpoint_height >= chunk_start_height && checkpoint_height <= chunk_end_height) - { - uint32_t index_in_chunk = checkpoint_height - chunk_start_height; - - if (index_in_chunk >= chunk_block_ids.size()) - { - logger(ERROR) << "CryptoNoteConfig.h checkpoint height " << checkpoint_height - << " index out of range in chunk " << chunk_index; - m_chunks.clear(); - return false; - } - - // VALIDATION: Verify that blockchain.dat matches the CryptoNoteConfig.h checkpoint - // (We validate against blockchain.dat, not the potentially overwritten DNS value) - const crypto::Hash& config_hash = checkpoint.second; - - // Re-fetch the original blockchain.dat hash for validation - std::vector original_block_ids = getBlockIdsFunc(checkpoint_height, 1); - if (original_block_ids.empty() || original_block_ids[0] != config_hash) - { - logger(ERROR) << "CryptoNoteConfig.h CHECKPOINT VALIDATION FAILED for chunk " - << chunk_index << " at height " << checkpoint_height << "!" - << " Expected (from CryptoNoteConfig.h): " << config_hash - << ", Got (from blockchain.dat): " << (original_block_ids.empty() ? NULL_HASH : original_block_ids[0]) - << ". Cannot create chunk - blockchain validation failed."; - m_chunks.clear(); - return false; - } - - // Replace with CryptoNoteConfig.h checkpoint hash (overwrites DNS if it was applied) - chunk_block_ids[index_in_chunk] = config_hash; - - // Update counters (only count if not already counted from DNS) - if (std::find(checkpoints_in_chunk.begin(), checkpoints_in_chunk.end(), checkpoint_height) == checkpoints_in_chunk.end()) - { - checkpoints_in_chunk.push_back(checkpoint_height); - } - else - { - // This checkpoint was already applied from DNS, now overwritten by CryptoNoteConfig.h - checkpoints_from_dns--; // Remove DNS count, add config count - } - checkpoints_from_config++; - - logger(DEBUGGING) << "Applied CryptoNoteConfig.h checkpoint hash for height " - << checkpoint_height << " in chunk " << chunk_index - << " (overwrites DNS if present)"; - } + m_chunks.clear(); + return false; } // Log validation success if there were checkpoints in this chunk - if (!checkpoints_in_chunk.empty()) + if (!checkpoint_result.checkpoints_in_chunk.empty()) { std::stringstream ss; - ss << "Validated and replaced " << checkpoints_in_chunk.size() << " checkpoint(s) in chunk " << chunk_index + ss << "Validated and replaced " << checkpoint_result.checkpoints_in_chunk.size() << " checkpoint(s) in chunk " << chunk_index << " (heights: "; - for (size_t i = 0; i < checkpoints_in_chunk.size(); i++) { + for (size_t i = 0; i < checkpoint_result.checkpoints_in_chunk.size(); i++) { if (i > 0) ss << ", "; - ss << checkpoints_in_chunk[i]; + ss << checkpoint_result.checkpoints_in_chunk[i]; } - ss << ") - Priority: " << checkpoints_from_config << " from CryptoNoteConfig.h, " - << checkpoints_from_dns << " from DNS, rest from blockchain.dat"; + ss << ") - Priority: " << checkpoint_result.checkpoints_from_config << " from CryptoNoteConfig.h, " + << checkpoint_result.checkpoints_from_dns << " from DNS, rest from blockchain.dat"; logger(INFO) << ss.str(); } @@ -586,93 +494,27 @@ namespace cn { return false; } - // PRIORITY ORDER for block hashes in chunk (applied in reverse order to maintain priority): - // 1. Start with blockchain.dat (base data - already loaded) - // 2. Apply DNS checkpoints (overwrites blockchain.dat) - // 3. Apply CryptoNoteConfig.h checkpoints (overwrites both DNS and blockchain.dat) - // Final priority: CryptoNoteConfig.h > DNS > blockchain.dat - - uint32_t checkpoints_from_config = 0; - uint32_t checkpoints_from_dns = 0; + // Apply checkpoint priority: CryptoNoteConfig.h > DNS > blockchain.dat + CheckpointApplicationResult checkpoint_result = apply_checkpoint_priority_to_chunk( + chunk_index, + chunk_start_height, + chunk_end_height, + chunk_block_ids, + getBlockIdsFunc, + false); // track_checkpoint_heights = false (only need counters) - // STEP 1: Apply DNS checkpoints (PRIORITY 2) - // These will be overwritten by CryptoNoteConfig.h if there's a conflict - for (const auto& checkpoint : m_dns_checkpoint_hashes) + if (!checkpoint_result.success) { - uint32_t checkpoint_height = checkpoint.first; - if (checkpoint_height >= chunk_start_height && checkpoint_height <= chunk_end_height) - { - uint32_t index_in_chunk = checkpoint_height - chunk_start_height; - - // VALIDATION: Verify that blockchain.dat matches the DNS checkpoint - const crypto::Hash& blockchain_hash = chunk_block_ids[index_in_chunk]; - const crypto::Hash& dns_hash = checkpoint.second; - - if (blockchain_hash != dns_hash) - { - logger(ERROR) << "DNS CHECKPOINT VALIDATION FAILED for chunk " - << chunk_index << " at height " << checkpoint_height << "!" - << " Expected (from DNS): " << dns_hash - << ", Got (from blockchain.dat): " << blockchain_hash; - return false; - } - - // Replace with DNS checkpoint hash - chunk_block_ids[index_in_chunk] = dns_hash; - checkpoints_from_dns++; - - logger(DEBUGGING) << "Applied DNS checkpoint hash for height " - << checkpoint_height << " in chunk " << chunk_index; - } - } - - // STEP 2: Apply CryptoNoteConfig.h checkpoints (PRIORITY 1 - HIGHEST) - // These overwrite both blockchain.dat and DNS checkpoints - for (const auto& checkpoint : m_old_checkpoint_hashes) - { - uint32_t checkpoint_height = checkpoint.first; - if (checkpoint_height >= chunk_start_height && checkpoint_height <= chunk_end_height) - { - uint32_t index_in_chunk = checkpoint_height - chunk_start_height; - const crypto::Hash& config_hash = checkpoint.second; - - // VALIDATION: Verify that blockchain.dat matches the CryptoNoteConfig.h checkpoint - // Re-fetch the original blockchain.dat hash for validation (before DNS overwrite) - std::vector original_block_ids = getBlockIdsFunc(checkpoint_height, 1); - if (original_block_ids.empty() || original_block_ids[0] != config_hash) - { - logger(ERROR) << "CryptoNoteConfig.h CHECKPOINT VALIDATION FAILED for chunk " - << chunk_index << " at height " << checkpoint_height << "!" - << " Expected (from CryptoNoteConfig.h): " << config_hash - << ", Got (from blockchain.dat): " << (original_block_ids.empty() ? NULL_HASH : original_block_ids[0]); - return false; - } - - // Replace with CryptoNoteConfig.h checkpoint hash (overwrites DNS if it was applied) - chunk_block_ids[index_in_chunk] = config_hash; - - // Update counters - if (checkpoints_from_dns > 0) - { - // Check if this height was in DNS (we can't easily track which specific heights, so we approximate) - // Actually, we can't know for sure without tracking, so we'll just count config checkpoints - // The logging will show the actual priority applied - } - checkpoints_from_config++; - - logger(DEBUGGING) << "Applied CryptoNoteConfig.h checkpoint hash for height " - << checkpoint_height << " in chunk " << chunk_index - << " (overwrites DNS if present)"; - } + return false; } - uint32_t total_checkpoints_applied = checkpoints_from_config + checkpoints_from_dns; + uint32_t total_checkpoints_applied = checkpoint_result.checkpoints_from_config + checkpoint_result.checkpoints_from_dns; if (total_checkpoints_applied > 0) { logger(INFO) << "Applied " << total_checkpoints_applied << " checkpoint(s) to chunk " << chunk_index - << " (Priority: " << checkpoints_from_config << " from CryptoNoteConfig.h, " - << checkpoints_from_dns << " from DNS, rest from blockchain.dat)"; + << " (Priority: " << checkpoint_result.checkpoints_from_config << " from CryptoNoteConfig.h, " + << checkpoint_result.checkpoints_from_dns << " from DNS, rest from blockchain.dat)"; } // Compute hash of this chunk's block IDs @@ -1450,6 +1292,86 @@ namespace cn { return (ip >> 16) & 0xFFFF; } + CheckpointList::PeerSamplingResult CheckpointList::sample_peers_with_diversity( + const std::vector& available_peers, + std::function getPeerNetwork16Func, + size_t num_peers_to_sample) + { + PeerSamplingResult result; + + if (available_peers.empty() || num_peers_to_sample == 0) + { + return result; + } + + // Limit sampling to available peers count + size_t actual_sample_size = std::min(num_peers_to_sample, available_peers.size()); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, available_peers.size() - 1); + + // Sample K peers with network diversity (max 1 vote per /16 network) + while (result.sampled_peers.size() < actual_sample_size) + { + // Try to find a peer from a different network + size_t attempts = 0; + const size_t max_attempts = available_peers.size() * 2; // Prevent infinite loop + + while (attempts < max_attempts && result.sampled_peers.size() < actual_sample_size) + { + size_t idx = dis(gen); + uint64_t peer_id = available_peers[idx]; + + // Check if we already sampled this peer + if (std::find(result.sampled_peers.begin(), result.sampled_peers.end(), peer_id) != result.sampled_peers.end()) + { + attempts++; + continue; + } + + // Check network diversity (max 1 vote per /16 network) + uint32_t net16 = getPeerNetwork16Func(peer_id); + if (result.network_votes[net16] >= 1 && result.sampled_peers.size() < available_peers.size()) + { + // Already have a vote from this network, try another peer + attempts++; + continue; + } + + // Accept this peer + result.sampled_peers.push_back(peer_id); + result.network_votes[net16] = std::min(result.network_votes[net16] + 1, 1U); + break; + } + + // If we couldn't find enough diverse peers, relax diversity requirement + if (result.sampled_peers.size() < actual_sample_size && attempts >= max_attempts) + { + fallback_peer_selection(available_peers, result.sampled_peers, actual_sample_size); + } + } + + return result; + } + + void CheckpointList::fallback_peer_selection( + const std::vector& available_peers, + std::vector& sampled_peers, + size_t target_size) + { + // Fall back to any available peer (diversity requirement relaxed) + for (uint64_t peer_id : available_peers) + { + if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) == sampled_peers.end()) + { + sampled_peers.push_back(peer_id); + if (sampled_peers.size() >= target_size) + break; + } + } + } + CheckpointList::HealthMetrics CheckpointList::get_health_metrics() const { HealthMetrics metrics; @@ -1641,58 +1563,19 @@ namespace cn { } // Apply priority order: CryptoNoteConfig.h > DNS > blockchain.dat - // Build a map of checkpoints to apply (same logic as chunk generation) - std::map> checkpoints_to_apply; - - // PRIORITY 1: CryptoNoteConfig.h checkpoints in this chunk - for (const auto& checkpoint : m_old_checkpoint_hashes) - { - uint32_t cp_height = checkpoint.first; - if (cp_height >= chunk_start_height && cp_height <= chunk_end_height) - { - checkpoints_to_apply[cp_height] = std::make_pair(checkpoint.second, "CryptoNoteConfig.h"); - } - } - - // PRIORITY 2: DNS checkpoints in this chunk (only if not in CryptoNoteConfig.h) - for (const auto& checkpoint : m_dns_checkpoint_hashes) + CheckpointApplicationResult checkpoint_result = apply_checkpoint_priority_to_chunk( + chunk_index, + chunk_start_height, + chunk_end_height, + chunk_block_ids, + nullptr, // No re-fetch needed - we already have the data + false); // No need to track specific heights + + if (!checkpoint_result.success) { - uint32_t cp_height = checkpoint.first; - if (cp_height >= chunk_start_height && cp_height <= chunk_end_height) - { - if (checkpoints_to_apply.find(cp_height) == checkpoints_to_apply.end()) - { - checkpoints_to_apply[cp_height] = std::make_pair(checkpoint.second, "DNS"); - } - } - } - - // Apply checkpoints with validation - for (const auto& cp_entry : checkpoints_to_apply) - { - uint32_t cp_height = cp_entry.first; - const crypto::Hash& cp_hash = cp_entry.second.first; - const std::string& cp_source = cp_entry.second.second; - - uint32_t index_in_chunk = cp_height - chunk_start_height; - - // Validate that blockchain.dat matches the checkpoint - const crypto::Hash& blockchain_hash = chunk_block_ids[index_in_chunk]; - - if (blockchain_hash != cp_hash) - { - logger(ERROR) << "CHECKPOINT VALIDATION FAILED: " - << "Checkpoint from " << cp_source << " at height " << cp_height - << " in chunk " << chunk_index << " does not match blockchain.dat!" - << " Expected: " << cp_hash - << ", Got: " << blockchain_hash; - result.is_valid = false; - result.first_mismatched_chunk_index = chunk_index; - return result; - } - - // Replace with checkpoint hash (using priority order) - chunk_block_ids[index_in_chunk] = cp_hash; + result.is_valid = false; + result.first_mismatched_chunk_index = chunk_index; + return result; } // Compute chunk hash using priority order @@ -1792,6 +1675,371 @@ namespace cn { return success; } + /** + * Apply checkpoint priority to chunk block IDs + * + * Applies checkpoints with priority order: CryptoNoteConfig.h > DNS > blockchain.dat + * Validates that blockchain.dat matches expected checkpoint values before applying. + */ + CheckpointList::CheckpointApplicationResult CheckpointList::apply_checkpoint_priority_to_chunk( + uint32_t chunk_index, + uint32_t chunk_start_height, + uint32_t chunk_end_height, + std::vector& chunk_block_ids, + std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc, + bool track_checkpoint_heights) const + { + CheckpointApplicationResult result; + result.success = true; + result.checkpoints_from_config = 0; + result.checkpoints_from_dns = 0; + + // STEP 1: Apply DNS checkpoints (PRIORITY 2) + // These will be overwritten by CryptoNoteConfig.h if there's a conflict + for (const auto& checkpoint : m_dns_checkpoint_hashes) + { + uint32_t checkpoint_height = checkpoint.first; + if (checkpoint_height >= chunk_start_height && checkpoint_height <= chunk_end_height) + { + uint32_t index_in_chunk = checkpoint_height - chunk_start_height; + + if (index_in_chunk >= chunk_block_ids.size()) + { + logger(ERROR) << "DNS checkpoint height " << checkpoint_height + << " index out of range in chunk " << chunk_index; + result.success = false; + return result; + } + + // VALIDATION: Verify that blockchain.dat matches the DNS checkpoint + const crypto::Hash& dns_hash = checkpoint.second; + + if (!validate_checkpoint_hash(checkpoint_height, dns_hash, chunk_block_ids, index_in_chunk, nullptr)) + { + logger(ERROR) << "DNS CHECKPOINT VALIDATION FAILED for chunk " + << chunk_index << " at height " << checkpoint_height << "!" + << " Expected (from DNS): " << dns_hash; + result.success = false; + return result; + } + + // Replace with DNS checkpoint hash + chunk_block_ids[index_in_chunk] = dns_hash; + result.checkpoints_from_dns++; + + if (track_checkpoint_heights) + { + result.checkpoints_in_chunk.push_back(checkpoint_height); + } + + logger(DEBUGGING) << "Applied DNS checkpoint hash for height " + << checkpoint_height << " in chunk " << chunk_index; + } + } + + // STEP 2: Apply CryptoNoteConfig.h checkpoints (PRIORITY 1 - HIGHEST) + // These overwrite both blockchain.dat and DNS checkpoints + for (const auto& checkpoint : m_old_checkpoint_hashes) + { + uint32_t checkpoint_height = checkpoint.first; + if (checkpoint_height >= chunk_start_height && checkpoint_height <= chunk_end_height) + { + uint32_t index_in_chunk = checkpoint_height - chunk_start_height; + + if (index_in_chunk >= chunk_block_ids.size()) + { + logger(ERROR) << "CryptoNoteConfig.h checkpoint height " << checkpoint_height + << " index out of range in chunk " << chunk_index; + result.success = false; + return result; + } + + // VALIDATION: Verify that blockchain.dat matches the CryptoNoteConfig.h checkpoint + const crypto::Hash& config_hash = checkpoint.second; + + if (!validate_checkpoint_hash(checkpoint_height, config_hash, chunk_block_ids, index_in_chunk, getBlockIdsFunc)) + { + logger(ERROR) << "CryptoNoteConfig.h CHECKPOINT VALIDATION FAILED for chunk " + << chunk_index << " at height " << checkpoint_height << "!" + << " Expected (from CryptoNoteConfig.h): " << config_hash; + result.success = false; + return result; + } + + // Replace with CryptoNoteConfig.h checkpoint hash (overwrites DNS if it was applied) + chunk_block_ids[index_in_chunk] = config_hash; + + // Update counters + if (track_checkpoint_heights) + { + // Check if this height was already in the list (from DNS) + auto it = std::find(result.checkpoints_in_chunk.begin(), result.checkpoints_in_chunk.end(), checkpoint_height); + if (it == result.checkpoints_in_chunk.end()) + { + result.checkpoints_in_chunk.push_back(checkpoint_height); + } + else + { + // This checkpoint was already applied from DNS, now overwritten by CryptoNoteConfig.h + result.checkpoints_from_dns--; // Remove DNS count, add config count + } + } + result.checkpoints_from_config++; + + logger(DEBUGGING) << "Applied CryptoNoteConfig.h checkpoint hash for height " + << checkpoint_height << " in chunk " << chunk_index + << " (overwrites DNS if present)"; + } + } + + return result; + } + + bool CheckpointList::validate_checkpoint_hash( + uint32_t checkpoint_height, + const crypto::Hash& expected_hash, + const std::vector& chunk_block_ids, + uint32_t index_in_chunk, + std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc) const + { + crypto::Hash actual_hash = NULL_HASH; + + // If getBlockIdsFunc is provided, re-fetch the original blockchain.dat hash for validation + // (This is needed because DNS may have already overwritten the value in chunk_block_ids) + if (getBlockIdsFunc) + { + std::vector original_block_ids = getBlockIdsFunc(checkpoint_height, 1); + if (!original_block_ids.empty()) + { + actual_hash = original_block_ids[0]; + } + } + else + { + // No re-fetch function provided - validate against current chunk_block_ids + // (This is used in validate_chunks_against_checkpoints where we already have the data) + if (index_in_chunk < chunk_block_ids.size()) + { + actual_hash = chunk_block_ids[index_in_chunk]; + } + } + + if (actual_hash == NULL_HASH) + { + logger(ERROR) << "Failed to get blockchain hash for checkpoint validation at height " << checkpoint_height; + return false; + } + + if (actual_hash != expected_hash) + { + logger(ERROR) << "Checkpoint validation failed at height " << checkpoint_height + << ": expected " << expected_hash + << ", got " << actual_hash; + return false; + } + + return true; + } + + /** + * Attempt to reach consensus on a chunk hash with peers + * + * Samples K peers with network diversity, collects their chunk hashes, + * and determines if M peers agree with the local hash. + * + * @param chunk_index The chunk index being validated + * @param local_chunk_hash The locally computed chunk hash + * @param getPeerChunkHashFunc Function to get chunk hash from a peer + * @param available_peers List of available peer IDs + * @param getPeerNetwork16Func Function to get /16 network for a peer + * @param req Consensus requirements (M, K, n) + * @param attempt_name Name of the attempt (for logging) + * @param total_null_hash_responses Reference to counter for NULL_HASH responses (updated) + * @param total_mismatches Reference to counter for mismatches (updated) + * @return AttemptResult with agreement count and consensus hash + */ + CheckpointList::AttemptResult CheckpointList::attempt_consensus_impl( + uint32_t chunk_index, + const crypto::Hash& local_chunk_hash, + std::function getPeerChunkHashFunc, + const std::vector& available_peers, + std::function getPeerNetwork16Func, + const ConsensusRequirements& req, + const std::string& attempt_name, + uint32_t& total_null_hash_responses, + uint32_t& total_mismatches) const + { + AttemptResult attempt_result; + attempt_result.agreements = 0; + attempt_result.consensus_hash = NULL_HASH; + + // Sample K peers (req.min_peers) with network diversity + PeerSamplingResult sampling_result = sample_peers_with_diversity( + available_peers, + getPeerNetwork16Func, + req.min_peers); + + const std::vector& sampled_peers = sampling_result.sampled_peers; + const std::map& network_votes = sampling_result.network_votes; + + // Check consensus: need M agreements from K sampled peers, with M agreements from at least n networks + // Track hash votes to find consensus hash (hash with M+ votes, if different from local) + // Use unordered_map since crypto::Hash doesn't have comparison operator for std::map + std::unordered_map> hash_votes; // hash -> vote count + // Track which networks agree with local hash (for network diversity requirement on M agreements) + std::map agreeing_networks_local; // network_16 -> count of agreeing peers + // Track which networks agree with consensus hash (if different from local) + // Use unordered_map since crypto::Hash doesn't have comparison operator for std::map + std::unordered_map, boost::hash> agreeing_networks_by_hash; // hash -> (network_16 -> count) + + uint32_t agreements = 0; + uint32_t null_hash_responses = 0; + uint32_t mismatches = 0; + + for (uint64_t peer_id : sampled_peers) + { + crypto::Hash peer_hash = getPeerChunkHashFunc(peer_id); + uint32_t net16 = getPeerNetwork16Func(peer_id); + + if (peer_hash == NULL_HASH) + { + // Peer didn't respond, timed out, or doesn't have this chunk in memory + // This is not necessarily a failure - the peer might not have created this chunk yet + // or might be using version 1 (doesn't support chunk-based checkpoints) + null_hash_responses++; + logger(INFO) << "Peer " << peer_id + << " returned NULL_HASH for chunk " << chunk_index + << " (" << attempt_name << ") - peer may not have this chunk in memory yet " + << "or may be using version 1 (doesn't support chunk checkpoints)"; + continue; // Don't count as agreement or disagreement - peer doesn't have the chunk + } + + // Count votes for this hash + hash_votes[peer_hash]++; + + if (peer_hash == local_chunk_hash) + { + agreements++; + agreeing_networks_local[net16]++; + logger(INFO) << "Peer " << peer_id << " agrees with local chunk " << chunk_index + << " hash (" << attempt_name << ") from network " << net16; + } + else + { + mismatches++; + agreeing_networks_by_hash[peer_hash][net16]++; + logger(WARNING) << "Peer " << peer_id + << " chunk hash mismatch for chunk " << chunk_index + << " (" << attempt_name << "): local=" << local_chunk_hash + << ", peer=" << peer_hash << " from network " << net16; + } + } + + // Find the consensus hash (hash with most votes, if >= M and different from local) + crypto::Hash consensus_hash = NULL_HASH; + uint32_t max_votes = 0; + for (const auto& vote : hash_votes) + { + if (vote.second >= req.min_agreements && vote.second > max_votes) + { + max_votes = vote.second; + consensus_hash = vote.first; + } + } + + // Only set consensus_hash if it's different from local and we have M+ agreements + if (consensus_hash != NULL_HASH && consensus_hash != local_chunk_hash && max_votes >= req.min_agreements) + { + attempt_result.consensus_hash = consensus_hash; + logger(WARNING) << "Chunk " << chunk_index << " (" << attempt_name + << "): M/K peers (" << max_votes << ") agree on different hash: " + << consensus_hash << " (local: " << local_chunk_hash << ")"; + } + + // CRITICAL: Verify that M agreeing peers come from at least n diverse networks + // This ensures consensus is not dominated by a single network + // Example: M=3, n=2 means we need 3 agreements from at least 2 different networks + // Valid: M11 M12 M21 (2 from network1, 1 from network2) or M11 M21 M22 (1 from network1, 2 from network2) + // Invalid: M11 M12 M13 (all 3 from network1 - not diverse enough) + uint32_t diverse_networks_in_agreements = 0; + bool meets_diversity_requirement = false; + + if (agreements >= req.min_agreements) + { + // Check if M agreeing peers come from at least n networks + diverse_networks_in_agreements = static_cast(agreeing_networks_local.size()); + meets_diversity_requirement = (diverse_networks_in_agreements >= req.min_diverse_networks); + + if (!meets_diversity_requirement) + { + logger(WARNING) << "Chunk " << chunk_index + << " consensus " << attempt_name << ": M=" << agreements + << " agreements but only from " << diverse_networks_in_agreements + << " network(s), need at least n=" << req.min_diverse_networks + << " networks for consensus"; + // Return 0 agreements if diversity requirement not met (consensus fails) + attempt_result.agreements = 0; + attempt_result.consensus_hash = NULL_HASH; + return attempt_result; + } + } + else if (consensus_hash != NULL_HASH && max_votes >= req.min_agreements) + { + // Check if M peers agreeing on consensus_hash come from at least n networks + auto it = agreeing_networks_by_hash.find(consensus_hash); + if (it != agreeing_networks_by_hash.end()) + { + diverse_networks_in_agreements = static_cast(it->second.size()); + meets_diversity_requirement = (diverse_networks_in_agreements >= req.min_diverse_networks); + + if (!meets_diversity_requirement) + { + logger(WARNING) << "Chunk " << chunk_index + << " consensus " << attempt_name << ": M=" << max_votes + << " peers agree on different hash but only from " << diverse_networks_in_agreements + << " network(s), need at least n=" << req.min_diverse_networks + << " networks for consensus"; + // Return 0 agreements if diversity requirement not met (consensus fails) + attempt_result.agreements = 0; + attempt_result.consensus_hash = NULL_HASH; + return attempt_result; + } + } + } + else + { + // Not enough agreements, but still track network diversity for logging + diverse_networks_in_agreements = static_cast(agreeing_networks_local.size()); + } + + // Also verify we sampled from at least n diverse networks (for K sampling requirement) + uint32_t diverse_networks_sampled = static_cast(network_votes.size()); + if (diverse_networks_sampled < req.min_diverse_networks) + { + logger(WARNING) << "Chunk " << chunk_index + << " consensus " << attempt_name << ": insufficient network diversity in sampling " + << "(sampled from " << diverse_networks_sampled << " networks, need n=" << req.min_diverse_networks << ")"; + // Return 0 agreements if diversity requirement not met (consensus fails) + attempt_result.agreements = 0; + attempt_result.consensus_hash = NULL_HASH; + return attempt_result; + } + + logger(INFO) << "Chunk " << chunk_index + << " consensus " << attempt_name << ": " << agreements + << "/" << sampled_peers.size() << " peers agreed (need M=" + << req.min_agreements << " from K=" << req.min_peers + << ", M agreements from n=" << diverse_networks_in_agreements + << " network(s), sampled from n=" << diverse_networks_sampled << " network(s))"; + + // Track statistics for better error messages + total_null_hash_responses += null_hash_responses; + total_mismatches += mismatches; + + attempt_result.agreements = agreements; + return attempt_result; + } + /** * Verify chunk with peer consensus (with second chance retry) * @@ -1842,172 +2090,21 @@ namespace cn { return result; } - // Helper function to sample peers and check consensus - // Returns: agreements count, and tracks if all responses were NULL_HASH + // Helper variables to track statistics across both attempts uint32_t total_null_hash_responses = 0; uint32_t total_mismatches = 0; - // Structure to return both agreement count and consensus hash - struct AttemptResult { - uint32_t agreements; - crypto::Hash consensus_hash; // Hash that M/K peers agreed on (if different from local, NULL_HASH otherwise) - }; - - auto attempt_consensus = [&](const std::string& attempt_name) -> AttemptResult { - AttemptResult attempt_result; - attempt_result.agreements = 0; - attempt_result.consensus_hash = NULL_HASH; - // Randomly sample M peers from available peers, ensuring network diversity - std::vector sampled_peers; - std::map network_votes; // network_16 -> vote_count (capped at 1) - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, available_peers.size() - 1); - - // Sample K peers (req.min_peers) with network diversity, need M agreements (req.min_agreements) - while (sampled_peers.size() < req.min_peers) - { - // Try to find a peer from a different network - size_t attempts = 0; - const size_t max_attempts = available_peers.size() * 2; // Prevent infinite loop - - while (attempts < max_attempts && sampled_peers.size() < req.min_peers) - { - size_t idx = dis(gen); - uint64_t peer_id = available_peers[idx]; - - // Check if we already sampled this peer - if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) != sampled_peers.end()) - { - attempts++; - continue; - } - - // Check network diversity (max 1 vote per /16 network) - uint32_t net16 = getPeerNetwork16Func(peer_id); - if (network_votes[net16] >= 1 && sampled_peers.size() < available_peers.size()) - { - // Already have a vote from this network, try another peer - attempts++; - continue; - } - - // Accept this peer - sampled_peers.push_back(peer_id); - network_votes[net16] = std::min(network_votes[net16] + 1, 1U); - break; - } - - // If we couldn't find enough diverse peers, relax diversity requirement - if (sampled_peers.size() < req.min_peers && attempts >= max_attempts) - { - // Fall back to any available peer (diversity requirement relaxed) - for (uint64_t peer_id : available_peers) - { - if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) == sampled_peers.end()) - { - sampled_peers.push_back(peer_id); - if (sampled_peers.size() >= req.min_peers) - break; - } - } - } - } - - // Check consensus: need M agreements from K sampled peers - // Track hash votes to find consensus hash (hash with M+ votes, if different from local) - // Use unordered_map since crypto::Hash doesn't have comparison operator for std::map - std::unordered_map> hash_votes; // hash -> vote count - uint32_t agreements = 0; - uint32_t null_hash_responses = 0; - uint32_t mismatches = 0; - - for (uint64_t peer_id : sampled_peers) - { - crypto::Hash peer_hash = getPeerChunkHashFunc(peer_id); - - if (peer_hash == NULL_HASH) - { - // Peer didn't respond, timed out, or doesn't have this chunk in memory - // This is not necessarily a failure - the peer might not have created this chunk yet - // or might be using version 1 (doesn't support chunk-based checkpoints) - null_hash_responses++; - logger(INFO) << "Peer " << peer_id - << " returned NULL_HASH for chunk " << chunk_index - << " (" << attempt_name << ") - peer may not have this chunk in memory yet " - << "or may be using version 1 (doesn't support chunk checkpoints)"; - continue; // Don't count as agreement or disagreement - peer doesn't have the chunk - } - - // Count votes for this hash - hash_votes[peer_hash]++; - - if (peer_hash == local_chunk_hash) - { - agreements++; - logger(INFO) << "Peer " << peer_id << " agrees with local chunk " << chunk_index - << " hash (" << attempt_name << ")"; - } - else - { - mismatches++; - logger(WARNING) << "Peer " << peer_id - << " chunk hash mismatch for chunk " << chunk_index - << " (" << attempt_name << "): local=" << local_chunk_hash - << ", peer=" << peer_hash; - } - } - - // Find the consensus hash (hash with most votes, if >= M and different from local) - crypto::Hash consensus_hash = NULL_HASH; - uint32_t max_votes = 0; - for (const auto& vote : hash_votes) - { - if (vote.second >= req.min_agreements && vote.second > max_votes) - { - max_votes = vote.second; - consensus_hash = vote.first; - } - } - - // Only set consensus_hash if it's different from local and we have M+ agreements - if (consensus_hash != NULL_HASH && consensus_hash != local_chunk_hash && max_votes >= req.min_agreements) - { - attempt_result.consensus_hash = consensus_hash; - logger(WARNING) << "Chunk " << chunk_index << " (" << attempt_name - << "): M/K peers (" << max_votes << ") agree on different hash: " - << consensus_hash << " (local: " << local_chunk_hash << ")"; - } - - // Verify we have at least n diverse networks - uint32_t diverse_networks = static_cast(network_votes.size()); - if (diverse_networks < req.min_diverse_networks) - { - logger(WARNING) << "Chunk " << chunk_index - << " consensus " << attempt_name << ": insufficient network diversity " - << "(have " << diverse_networks << " networks, need n=" << req.min_diverse_networks << ")"; - // Return 0 agreements if diversity requirement not met (consensus fails) - attempt_result.agreements = 0; - attempt_result.consensus_hash = NULL_HASH; - return attempt_result; - } - - logger(INFO) << "Chunk " << chunk_index - << " consensus " << attempt_name << ": " << agreements - << "/" << sampled_peers.size() << " peers agreed (need M=" - << req.min_agreements << " from K=" << req.min_peers - << ", have n=" << diverse_networks << " diverse networks)"; - - // Track statistics for better error messages - total_null_hash_responses += null_hash_responses; - total_mismatches += mismatches; - - attempt_result.agreements = agreements; - return attempt_result; - }; - // First attempt - AttemptResult first_attempt = attempt_consensus("first attempt"); + AttemptResult first_attempt = attempt_consensus_impl( + chunk_index, + local_chunk_hash, + getPeerChunkHashFunc, + available_peers, + getPeerNetwork16Func, + req, + "first attempt", + total_null_hash_responses, + total_mismatches); result.agreements_first_attempt = first_attempt.agreements; result.consensus_hash_first_attempt = first_attempt.consensus_hash; @@ -2041,7 +2138,16 @@ namespace cn { } result.used_second_chance = true; - AttemptResult second_attempt = attempt_consensus("second attempt"); + AttemptResult second_attempt = attempt_consensus_impl( + chunk_index, + local_chunk_hash, + getPeerChunkHashFunc, + available_peers, + getPeerNetwork16Func, + req, + "second attempt", + total_null_hash_responses, + total_mismatches); result.agreements_second_attempt = second_attempt.agreements; result.consensus_hash_second_attempt = second_attempt.consensus_hash; diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index bc7f5d2c0..0564ace39 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -33,6 +33,7 @@ #include "P2p/LevinProtocol.h" #include // for UINT64_MAX #include "CryptoNoteCore/CheckpointList.h" +#include "CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h" using namespace logging; using namespace common; @@ -68,12 +69,14 @@ CryptoNoteProtocolHandler::CryptoNoteProtocolHandler(const Currency ¤cy, p m_peersCount(0), logger(log, "protocol"), m_dispatcher(dispatcher), - m_maxObjectCount(cn::COMMAND_RPC_GET_OBJECTS_MAX_COUNT), - m_current_validating_chunk_index(UINT32_MAX), - m_last_chunk_validation_attempt(0) + m_maxObjectCount(cn::COMMAND_RPC_GET_OBJECTS_MAX_COUNT) { if (!m_p2p) m_p2p = &m_p2p_stub; + + // Initialize chunk validation manager + m_chunkValidationManager = std::unique_ptr( + new ChunkValidationManager(m_core, m_p2p, m_currency, log, m_peersCount, m_stop)); } size_t CryptoNoteProtocolHandler::getPeerCount() const @@ -679,9 +682,9 @@ bool CryptoNoteProtocolHandler::on_idle() if (cn::P2P_CURRENT_VERSION >= cn::P2P_CHECKPOINT_LIST_VERSION) { // Check pending validations for consensus (asynchronous approach) - check_pending_chunk_validations(); + m_chunkValidationManager->check_pending_chunk_validations(); // Start new validations if needed - validate_unverified_chunks(); + m_chunkValidationManager->validate_unverified_chunks(); } return m_core.on_idle(); @@ -1340,40 +1343,18 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE logger(INFO) << context << " Received chunk hash response from peer " << peer_id << " for chunk " << arg.chunk_index; // Check if we're still validating this chunk (late responses might arrive after timeout) - bool still_validating = false; - { - std::lock_guard lock(m_chunk_validation_mutex); - still_validating = (m_current_validating_chunk_index == arg.chunk_index); - } + bool still_validating = m_chunkValidationManager->is_chunk_being_validated(arg.chunk_index); // Store the response in pending chunk hashes map for the consensus mechanism // NOTE: NULL_HASH is a valid response (means peer doesn't have this chunk) - uint64_t response_time = time(nullptr); - bool was_pending = false; - { - std::lock_guard lock(m_pending_chunk_hashes_mutex); - auto key = std::make_pair(peer_id, arg.chunk_index); - was_pending = (m_pending_chunk_hashes.find(key) != m_pending_chunk_hashes.end()); - ChunkHashResponse response; - response.hash = arg.chunk_hash; - response.timestamp = response_time; - m_pending_chunk_hashes[key] = response; - } + m_chunkValidationManager->store_chunk_hash_response(peer_id, arg.chunk_index, arg.chunk_hash); // Log if this is a late response (arrived after timeout) - if (!still_validating && was_pending) - { - logger(INFO) << context << " Received LATE chunk hash response from peer " << peer_id - << " for chunk " << arg.chunk_index - << " (validation already completed - response arrived after 5s timeout). " - << "Response cached and will be used in next validation attempt."; - } - else if (!still_validating && !was_pending) + if (!still_validating) { - // Response arrived for a request we already gave up on - this is fine, we'll use it next time logger(DEBUGGING) << context << " Received chunk hash response from peer " << peer_id << " for chunk " << arg.chunk_index - << " (from previous request, will be used in next validation attempt)"; + << " (validation may have completed - response will be used in next validation attempt)"; } if (arg.chunk_hash == NULL_HASH) @@ -1389,40 +1370,6 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE return 1; } -bool CryptoNoteProtocolHandler::send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index) -{ - // Find the peer connection - CryptoNoteConnectionContext* peer_context = nullptr; - m_p2p->for_each_connection([&peer_context, peer_id](CryptoNoteConnectionContext& ctx, uint64_t id) { - if (id == peer_id) { - peer_context = &ctx; - } - }); - - if (!peer_context) { - logger(WARNING) << "Cannot send async chunk hash request to peer " << peer_id << ": peer not found"; - return false; - } - - // Send request (non-blocking) - NOTIFY_REQUEST_CHUNK_HASH::request req; - req.chunk_index = chunk_index; - - logger(INFO) << "Sending async chunk hash request for chunk " << chunk_index << " to peer " << peer_id - << " " << *peer_context; - - bool sent = post_notify(*m_p2p, req, *peer_context); - if (!sent) { - logger(WARNING) << "Failed to send async chunk hash request to peer " << peer_id << " " << *peer_context - << " for chunk " << chunk_index; - return false; - } - - logger(INFO) << "Successfully sent async chunk hash request for chunk " << chunk_index - << " to peer " << peer_id << " " << *peer_context; - return true; -} - crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t peer_id, uint32_t chunk_index) { // Find the peer connection @@ -1448,89 +1395,10 @@ crypto::Hash CryptoNoteProtocolHandler::request_chunk_hash_from_peer(uint64_t pe return NULL_HASH; } - // Check if we already have a cached response from a previous request (late response) - // This allows us to use responses that arrived after the timeout - { - std::lock_guard lock(m_pending_chunk_hashes_mutex); - auto key = std::make_pair(peer_id, chunk_index); - auto it = m_pending_chunk_hashes.find(key); - if (it != m_pending_chunk_hashes.end()) { - crypto::Hash cached_result = it->second.hash; - m_pending_chunk_hashes.erase(it); - logger(INFO) << "Using cached chunk hash response from peer " << peer_id << " " << *peer_context - << " for chunk " << chunk_index << " (from previous request): " << cached_result; - return cached_result; - } - // Log for debugging - show what pending entries exist - if (!m_pending_chunk_hashes.empty()) { - logger(DEBUGGING) << "No cached response for peer " << peer_id << " chunk " << chunk_index - << ". Total pending entries: " << m_pending_chunk_hashes.size(); - // Log first few pending entries to see what we have - int count = 0; - for (const auto& entry : m_pending_chunk_hashes) { - if (count++ < 3) { - logger(DEBUGGING) << " Pending: peer " << entry.first.first << " chunk " << entry.first.second - << " hash " << entry.second.hash << " (received at " << entry.second.timestamp << ")"; - } - } - } else { - logger(DEBUGGING) << "No cached response for peer " << peer_id << " chunk " << chunk_index - << " (no pending entries)"; - } - } - - // Send request - NOTIFY_REQUEST_CHUNK_HASH::request req; - req.chunk_index = chunk_index; - - logger(INFO) << "Requesting chunk hash for chunk " << chunk_index << " from peer " << peer_id - << " " << *peer_context - << " (connection_id: " << peer_context->m_connection_id - << ", state: " << get_protocol_state_string(peer_context->m_state) - << ", version: " << static_cast(peer_context->version) << ")"; - - bool sent = post_notify(*m_p2p, req, *peer_context); - if (!sent) { - logger(WARNING) << "Failed to send chunk hash request to peer " << peer_id << " " << *peer_context - << " for chunk " << chunk_index; - // Remove the pending entry we just created - { - std::lock_guard lock(m_pending_chunk_hashes_mutex); - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } - return NULL_HASH; - } - - logger(INFO) << "Successfully sent chunk hash request for chunk " << chunk_index - << " to peer " << peer_id << " " << *peer_context << " (waiting for response...)"; - - // Wait for response (with short timeout) - // Use a short timeout (5 seconds) to avoid blocking the P2P message queue - // If response arrives later, it will be stored in m_pending_chunk_hashes and used in next validation attempt - const int max_wait_ms = 5000; // 5 second timeout - responses arriving later will be used in next attempt - const int poll_interval_ms = 50; - int waited_ms = 0; - - while (waited_ms < max_wait_ms) { - std::this_thread::sleep_for(std::chrono::milliseconds(poll_interval_ms)); - waited_ms += poll_interval_ms; - - std::lock_guard lock(m_pending_chunk_hashes_mutex); - auto it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); - if (it != m_pending_chunk_hashes.end()) { - crypto::Hash result = it->second.hash; - m_pending_chunk_hashes.erase(it); - logger(INFO) << "Received chunk hash response from peer " << peer_id << " " << *peer_context - << " for chunk " << chunk_index << ": " << result; - return result; - } - } - - // Timeout reached - response may arrive later and will be used in next validation attempt - // The response handler will store it in m_pending_chunk_hashes when it arrives - logger(DEBUGGING) << "Timeout waiting for immediate chunk hash response from peer " << peer_id - << " " << *peer_context << " for chunk " << chunk_index - << " (response may arrive later and will be used in next validation attempt)"; + // NOTE: This function is deprecated - we now use async validation via ChunkValidationManager + // This synchronous version is kept for backward compatibility but should not be used + // Responses will be handled asynchronously by the ChunkValidationManager + logger(WARNING) << "request_chunk_hash_from_peer is deprecated - use async validation instead"; return NULL_HASH; } @@ -1553,519 +1421,4 @@ int CryptoNoteProtocolHandler::handle_response_checkpoint_list(int command, NOTI return 1; } -bool CryptoNoteProtocolHandler::validate_unverified_chunks() -{ - // Check if we have unverified chunks first (fast check) - std::vector unverified_chunks = m_core.getCheckpointList().get_unverified_chunks(); - if (unverified_chunks.empty()) - { - return false; // No unverified chunks - nothing to validate - } - - // We need at least some peers to validate (but don't require full synchronization) - // Validation can happen during sync as long as we have peers - if (m_peersCount.load() == 0) - { - logger(DEBUGGING) << "Cannot validate chunks: no peers connected yet"; - return false; - } - - // Rate limit: don't attempt validation more than once every P2P_CHECKPOINT_LIST_RE_REQUEST seconds (5 minutes) - // EXCEPTION: Allow immediate attempt if this is the first time we have unverified chunks - // (bypass rate limit on first attempt after chunks are created) - uint64_t time_now = time(nullptr); - bool bypass_rate_limit = false; - { - std::lock_guard lock(m_chunk_validation_mutex); - - // Check if this is the first validation attempt (m_last_chunk_validation_attempt == 0) - // or if enough time has passed - uint64_t time_since_last = time_now - m_last_chunk_validation_attempt; - - if (m_last_chunk_validation_attempt == 0) - { - // First validation attempt - allow it immediately - bypass_rate_limit = true; - logger(INFO) << "First chunk validation attempt - bypassing rate limit"; - } - else if (time_since_last < cn::P2P_CHECKPOINT_LIST_RE_REQUEST) - { - // Too soon since last attempt - logger(DEBUGGING) << "Chunk validation rate limited: " - << (cn::P2P_CHECKPOINT_LIST_RE_REQUEST - time_since_last) - << " seconds remaining"; - return false; - } - - m_last_chunk_validation_attempt = time_now; - } - - // Sort to ensure chronological order (oldest first) - std::sort(unverified_chunks.begin(), unverified_chunks.end()); - - logger(INFO) << "Attempting to validate " << unverified_chunks.size() - << " unverified chunk(s) via P2P consensus"; - - // Check if we're already validating a chunk - { - std::lock_guard lock(m_chunk_validation_mutex); - if (m_current_validating_chunk_index != UINT32_MAX) - { - // Already validating a chunk - wait for it to complete - return false; - } - } - - // Get current blockchain height to calculate peer uptime in blocks - uint32_t current_height = get_current_blockchain_height(); - uint32_t block_time = m_currency.difficultyTarget(); // Block time in seconds (120 for mainnet/testnet) - uint32_t min_uptime_blocks = m_core.getCheckpointList().get_min_peer_uptime_blocks(); - - // Find eligible peers (meet uptime requirement and support chunk-based checkpoints) - std::vector eligible_peers; - std::map peer_network_16; // peer_id -> /16 network - - // First, log all connected peers for debugging - uint32_t total_connected = 0; - m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { - total_connected++; - }); - logger(INFO) << "Checking " << total_connected << " connected peer(s) for chunk validation eligibility"; - - m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { - // Only consider peers that support chunk-based checkpoints - if (ctx.version < cn::P2P_CHECKPOINT_LIST_VERSION) - { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: version " - << static_cast(ctx.version) << " < " << cn::P2P_CHECKPOINT_LIST_VERSION; - return; // Skip old version peers - } - - // Accept peers in normal state (synchronized) OR synchronizing state - // We can validate chunks even during sync, as long as peers are connected - if (ctx.m_state != CryptoNoteConnectionContext::state_normal && - ctx.m_state != CryptoNoteConnectionContext::state_synchronizing) - { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: state " - << get_protocol_state_string(ctx.m_state) << " (need normal or synchronizing)"; - return; // Skip peers that aren't in a usable state - } - - // Calculate peer uptime in blocks - // Uptime = (current_time - connection_start_time) / block_time - // This is an approximation - ideally we'd track peer's blockchain height when they first connected - time_t connection_duration = time_now - ctx.m_started; - if (connection_duration < 0) - { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: invalid connection time"; - return; // Invalid connection time - } - - uint32_t peer_uptime_blocks = static_cast(connection_duration / block_time); - - // Check if peer meets minimum uptime requirement - if (peer_uptime_blocks < min_uptime_blocks) - { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: uptime " - << peer_uptime_blocks << " blocks < " << min_uptime_blocks << " blocks required"; - return; // Peer doesn't meet uptime requirement - } - - // Peer is eligible - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " is ELIGIBLE: version " - << static_cast(ctx.version) << ", state " - << get_protocol_state_string(ctx.m_state) << ", uptime " - << peer_uptime_blocks << " blocks"; - eligible_peers.push_back(peer_id); - peer_network_16[peer_id] = CheckpointList::get_network_16(ctx.m_remote_ip); - }); - - if (eligible_peers.empty()) - { - logger(INFO) << "No eligible peers for chunk validation (need uptime > " - << min_uptime_blocks << " blocks, version 2+, and in normal/synchronizing state)"; - logger(INFO) << "Note: Only ACTIVE CONNECTIONS are considered, not peers in peerlist. " - << "Use 'print_cn' command to see active connections. " - << "To force connection to a peer, use --add-priority-node :"; - logger(INFO) << "Chunk validation will be retried when eligible peers become available"; - return false; - } - - logger(INFO) << "Found " << eligible_peers.size() << " eligible peer(s) for chunk validation " - << "(uptime > " << min_uptime_blocks << " blocks, support version 2+)"; - - // Validate chunks in chronological order (oldest first) - // This ensures we catch divergences at the root cause - for (uint32_t chunk_index : unverified_chunks) - { - // Check if we should stop (node shutting down) - if (m_stop.load()) - { - break; - } - - // Mark this chunk as being validated - { - std::lock_guard lock(m_chunk_validation_mutex); - if (m_current_validating_chunk_index != UINT32_MAX) - { - // Another thread started validation - break; - } - m_current_validating_chunk_index = chunk_index; - } - - // Get local chunk hash - crypto::Hash local_chunk_hash = m_core.getCheckpointList().get_chunk_hash(chunk_index); - if (local_chunk_hash == NULL_HASH) - { - logger(WARNING) << "Cannot validate chunk " << chunk_index << ": chunk hash not found in memory"; - { - std::lock_guard lock(m_chunk_validation_mutex); - m_current_validating_chunk_index = UINT32_MAX; - } - continue; - } - - logger(INFO) << "Validating chunk " << chunk_index << " (oldest unverified chunk) " - << "with " << eligible_peers.size() << " eligible peer(s)"; - - // Check if this chunk is already being validated asynchronously - { - std::lock_guard lock(m_pending_validations_mutex); - if (m_pending_validations.find(chunk_index) != m_pending_validations.end()) - { - logger(DEBUGGING) << "Chunk " << chunk_index << " is already being validated asynchronously, skipping"; - { - std::lock_guard lock2(m_chunk_validation_mutex); - m_current_validating_chunk_index = UINT32_MAX; - } - continue; - } - } - - // Calculate consensus requirements (M, K, n) - CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(eligible_peers.size()); - logger(INFO) << "Consensus requirements (testnet): M=" << req.min_agreements - << ", K=" << req.min_peers << ", n=" << req.min_diverse_networks - << " (have " << eligible_peers.size() << " available peers)"; - - // Sample K peers with network diversity (same logic as verify_chunk_with_peer_consensus) - std::vector sampled_peers; - std::map network_votes; // network_16 -> vote_count (capped at 1) - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, eligible_peers.size() - 1); - - // Sample K peers (req.min_peers) with network diversity - while (sampled_peers.size() < req.min_peers && sampled_peers.size() < eligible_peers.size()) - { - size_t attempts = 0; - const size_t max_attempts = eligible_peers.size() * 2; - - while (attempts < max_attempts && sampled_peers.size() < req.min_peers) - { - size_t idx = dis(gen); - uint64_t peer_id = eligible_peers[idx]; - - // Check if we already sampled this peer - if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) != sampled_peers.end()) - { - attempts++; - continue; - } - - // Check network diversity (max 1 vote per /16 network) - uint32_t net16 = peer_network_16[peer_id]; - if (network_votes[net16] >= 1 && sampled_peers.size() < eligible_peers.size()) - { - attempts++; - continue; - } - - // Accept this peer - sampled_peers.push_back(peer_id); - network_votes[net16] = std::min(network_votes[net16] + 1, 1U); - break; - } - - // If we couldn't find enough diverse peers, relax diversity requirement - if (sampled_peers.size() < req.min_peers && attempts >= max_attempts) - { - for (uint64_t peer_id : eligible_peers) - { - if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) == sampled_peers.end()) - { - sampled_peers.push_back(peer_id); - if (sampled_peers.size() >= req.min_peers) - break; - } - } - } - } - - if (sampled_peers.empty()) - { - logger(WARNING) << "Could not sample any peers for chunk " << chunk_index << " validation"; - { - std::lock_guard lock(m_chunk_validation_mutex); - m_current_validating_chunk_index = UINT32_MAX; - } - continue; - } - - // Verify we have at least n distinct networks (network diversity requirement) - std::set distinct_networks; - for (uint64_t peer_id : sampled_peers) - { - distinct_networks.insert(peer_network_16[peer_id]); - } - - if (distinct_networks.size() < req.min_diverse_networks) - { - logger(WARNING) << "Could not achieve network diversity for chunk " << chunk_index - << " validation: have " << distinct_networks.size() - << " distinct networks, need " << req.min_diverse_networks - << ". Sampled " << sampled_peers.size() << " peer(s)."; - { - std::lock_guard lock(m_chunk_validation_mutex); - m_current_validating_chunk_index = UINT32_MAX; - } - continue; - } - - logger(INFO) << "Sampled " << sampled_peers.size() << " peer(s) from " << distinct_networks.size() - << " distinct network(s) (requirement: " << req.min_diverse_networks << " networks)"; - - // Send async requests to sampled peers - uint64_t request_time = time(nullptr); - uint32_t requests_sent = 0; - for (uint64_t peer_id : sampled_peers) - { - if (send_chunk_hash_request_async(peer_id, chunk_index)) - { - requests_sent++; - } - } - - if (requests_sent == 0) - { - logger(WARNING) << "Failed to send any async requests for chunk " << chunk_index; - { - std::lock_guard lock(m_chunk_validation_mutex); - m_current_validating_chunk_index = UINT32_MAX; - } - continue; - } - - // Store pending validation state - { - std::lock_guard lock(m_pending_validations_mutex); - PendingChunkValidation pending; - pending.chunk_index = chunk_index; - pending.request_timestamp = request_time; - pending.attempt_start_time = request_time; - pending.attempt_number = 1; - pending.requested_peers = sampled_peers; - pending.local_hash = local_chunk_hash; - pending.is_first_attempt = true; - m_pending_validations[chunk_index] = pending; - } - - logger(INFO) << "Sent async chunk hash requests for chunk " << chunk_index - << " to " << requests_sent << " peer(s). " - << "Will check for consensus after 2 minutes."; - - // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) - { - std::lock_guard lock(m_chunk_validation_mutex); - m_current_validating_chunk_index = UINT32_MAX; - } - - // Don't process next chunk until this one is validated (file must be sequential) - // The check_pending_chunk_validations() function will handle consensus checking - break; - } - - return true; -} - -void CryptoNoteProtocolHandler::check_pending_chunk_validations() -{ - uint64_t time_now = time(nullptr); - const uint64_t CONSENSUS_WAIT_SECONDS = 120; // 2 minutes - const uint64_t RETRY_DELAY_SECONDS = 60; // 1 minute delay before retry - - std::lock_guard lock(m_pending_validations_mutex); - - // Iterate through pending validations - for (auto it = m_pending_validations.begin(); it != m_pending_validations.end();) - { - uint32_t chunk_index = it->first; - PendingChunkValidation& pending = it->second; - - uint64_t elapsed = time_now - pending.request_timestamp; - - // Check if 2 minutes have passed since requests were sent - if (elapsed < CONSENSUS_WAIT_SECONDS) - { - // Not enough time has passed yet, skip this validation - ++it; - continue; - } - - // 2 minutes have passed - check for consensus - logger(INFO) << "Checking consensus for chunk " << chunk_index - << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; - - // Collect responses from requested peers - std::unordered_map> hash_votes; // hash -> vote count - uint32_t agreements = 0; - uint32_t null_hash_responses = 0; - uint32_t responses_received = 0; - - { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - - for (uint64_t peer_id : pending.requested_peers) - { - auto response_it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); - if (response_it != m_pending_chunk_hashes.end()) - { - crypto::Hash peer_hash = response_it->second.hash; - responses_received++; - - if (peer_hash == NULL_HASH) - { - null_hash_responses++; - continue; - } - - // Count votes for this hash - hash_votes[peer_hash]++; - - if (peer_hash == pending.local_hash) - { - agreements++; - } - } - } - } - - // Calculate consensus requirements - CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(pending.requested_peers.size()); - - logger(INFO) << "Chunk " << chunk_index << " consensus check: received " << responses_received - << " response(s) from " << pending.requested_peers.size() << " requested peer(s). " - << "Agreements: " << agreements << " (need M=" << req.min_agreements << "), " - << "NULL_HASH responses: " << null_hash_responses; - - // Check if we have M agreements (consensus reached) - if (agreements >= req.min_agreements) - { - // Consensus reached - save to checkpoint.dat - logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index - << " validated via peer consensus (" << agreements - << " agreements, need M=" << req.min_agreements << ")"; - - if (m_core.getCheckpointList().add_verified_chunk_to_file(chunk_index)) - { - logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index - << " saved to checkpoint.dat"; - - // Remove pending validation - it = m_pending_validations.erase(it); - - // Clean up responses for this chunk - { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - for (uint64_t peer_id : pending.requested_peers) - { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } - } - continue; - } - else - { - logger(ERROR) << "Failed to save validated chunk " << chunk_index << " to checkpoint.dat"; - // Remove pending validation anyway (we'll retry later) - it = m_pending_validations.erase(it); - continue; - } - } - - // Consensus not reached - check if we should retry - if (pending.is_first_attempt) - { - // First attempt failed - wait 1 more minute (3 minutes total) before retry - uint64_t total_elapsed = time_now - pending.attempt_start_time; - if (total_elapsed < (CONSENSUS_WAIT_SECONDS + RETRY_DELAY_SECONDS)) - { - // Still waiting for retry delay - ++it; - continue; - } - - // Retry delay passed - start second attempt - logger(INFO) << "Chunk " << chunk_index - << " consensus failed on first attempt. Starting second attempt..."; - - // Send new requests to same peers (or get new eligible peers) - // For now, reuse same peers - uint64_t retry_time = time_now; - uint32_t requests_sent = 0; - for (uint64_t peer_id : pending.requested_peers) - { - if (send_chunk_hash_request_async(peer_id, chunk_index)) - { - requests_sent++; - } - } - - if (requests_sent > 0) - { - // Update pending validation for second attempt - pending.request_timestamp = retry_time; - pending.attempt_number = 2; - pending.is_first_attempt = false; - - logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index - << " to " << requests_sent << " peer(s). " - << "Will check for consensus after 2 minutes."; - } - else - { - logger(WARNING) << "Failed to send second attempt requests for chunk " << chunk_index; - // Remove pending validation - it = m_pending_validations.erase(it); - continue; - } - } - else - { - // Second attempt also failed - consensus failed - logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index - << " validation FAILED: peer consensus did not agree with local chunk hash " - << "after 2 attempts. This indicates blockchain divergence."; - - // TODO: Handle rollback (same logic as before) - // For now, just remove pending validation - it = m_pending_validations.erase(it); - - // Clean up responses - { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - for (uint64_t peer_id : pending.requested_peers) - { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } - } - continue; - } - - ++it; - } -} - }; // namespace cn diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h index d4f57dfd7..d2450d9c9 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.h @@ -9,6 +9,7 @@ #include #include +#include #include #include "../CryptoNoteConfig.h" @@ -18,6 +19,7 @@ #include "CryptoNoteProtocol/CryptoNoteProtocolHandlerCommon.h" #include "CryptoNoteProtocol/ICryptoNoteProtocolObserver.h" #include "CryptoNoteProtocol/ICryptoNoteProtocolQuery.h" +#include "CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h" #include "P2p/P2pProtocolDefinitions.h" #include "P2p/NetNodeCommon.h" @@ -25,6 +27,7 @@ #include + namespace platform_system { class Dispatcher; } @@ -134,47 +137,7 @@ namespace cn std::atomic m_maxObjectCount; - // Pending chunk hash responses: (peer_id, chunk_index) -> (hash, timestamp) - // Used for asynchronous consensus mechanism - struct ChunkHashResponse { - crypto::Hash hash; - uint64_t timestamp; // When response was received - }; - mutable std::mutex m_pending_chunk_hashes_mutex; - std::map, ChunkHashResponse> m_pending_chunk_hashes; - - // Pending chunk validation attempts: chunk_index -> validation state - struct PendingChunkValidation { - uint32_t chunk_index; - uint64_t request_timestamp; // When requests were sent - uint64_t attempt_start_time; // When this attempt started - uint32_t attempt_number; // 1 or 2 - std::vector requested_peers; // Peers we requested from - crypto::Hash local_hash; - bool is_first_attempt; - }; - mutable std::mutex m_pending_validations_mutex; - std::map m_pending_validations; // chunk_index -> validation state - - // Chunk validation state: track which chunk we're currently validating - // This prevents duplicate validation attempts and ensures chronological order - mutable std::mutex m_chunk_validation_mutex; - uint32_t m_current_validating_chunk_index; // Currently validating chunk (or UINT32_MAX if none) - uint64_t m_last_chunk_validation_attempt; // Last time we attempted validation (to avoid spamming) - - // Validate unverified chunks in chronological order (oldest first) - // This ensures we catch divergences at the root cause and rollback to the correct point - // Only validates when peers meet uptime requirements - // @return true if validation was attempted (even if it failed) - bool validate_unverified_chunks(); - - // Check pending chunk validations for consensus (called periodically) - // Checks if we have M identical hashes within the time window (2 minutes) - // If consensus reached, processes it; if timeout, schedules retry - void check_pending_chunk_validations(); - - // Send chunk hash request asynchronously (non-blocking) - // Returns true if request was sent successfully - bool send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index); + // Chunk validation manager (handles asynchronous P2P consensus) + std::unique_ptr m_chunkValidationManager; }; } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp new file mode 100644 index 000000000..befc0de28 --- /dev/null +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -0,0 +1,691 @@ +// Copyright (c) 2011-2017 The Cryptonote developers +// Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs +// Copyright (c) 2018-2023 Conceal Network & Conceal Devs +// +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "CryptoNoteProtocolHandlerChunk.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "CryptoNoteCore/CheckpointList.h" +#include "CryptoNoteCore/Currency.h" +#include "CryptoNoteProtocol/CryptoNoteProtocolDefinitions.h" +#include "CryptoNoteProtocol/CryptoNoteProtocolHandler.h" +#include "P2p/ConnectionContext.h" +#include "P2p/LevinProtocol.h" + +using namespace logging; +using namespace common; + +namespace cn +{ + +namespace +{ + // Helper template function to send notifications (same as in CryptoNoteProtocolHandler.cpp) + template + bool post_notify(IP2pEndpoint &p2p, typename t_parametr::request &arg, const CryptoNoteConnectionContext &context) + { + return p2p.invoke_notify_to_peer(t_parametr::ID, LevinProtocol::encode(arg), context); + } +} // namespace + +ChunkValidationManager::ChunkValidationManager(ICore& core, + IP2pEndpoint* p2p, + const Currency& currency, + logging::ILogger& log, + std::atomic& peersCount, + std::atomic& stop) + : m_core(core) + , m_p2p(p2p) + , m_currency(currency) + , logger(log, "chunk_validation") + , m_peersCount(peersCount) + , m_stop(stop) + , m_current_validating_chunk_index(UINT32_MAX) + , m_last_chunk_validation_attempt(0) +{ +} + +bool ChunkValidationManager::send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index) +{ + // Find the peer connection + CryptoNoteConnectionContext* peer_context = nullptr; + m_p2p->for_each_connection([&peer_context, peer_id](CryptoNoteConnectionContext& ctx, uint64_t id) { + if (id == peer_id) { + peer_context = &ctx; + } + }); + + if (!peer_context) { + logger(WARNING) << "Cannot send async chunk hash request to peer " << peer_id << ": peer not found"; + return false; + } + + // Send request (non-blocking) + NOTIFY_REQUEST_CHUNK_HASH::request req; + req.chunk_index = chunk_index; + + logger(INFO) << "Sending async chunk hash request for chunk " << chunk_index << " to peer " << peer_id + << " " << *peer_context; + + bool sent = post_notify(*m_p2p, req, *peer_context); + if (!sent) { + logger(WARNING) << "Failed to send async chunk hash request to peer " << peer_id << " " << *peer_context + << " for chunk " << chunk_index; + return false; + } + + logger(INFO) << "Successfully sent async chunk hash request for chunk " << chunk_index + << " to peer " << peer_id << " " << *peer_context; + return true; +} + +void ChunkValidationManager::store_chunk_hash_response(uint64_t peer_id, uint32_t chunk_index, const crypto::Hash& hash) +{ + uint64_t response_time = time(nullptr); + { + std::lock_guard lock(m_pending_chunk_hashes_mutex); + auto key = std::make_pair(peer_id, chunk_index); + ChunkHashResponse response; + response.hash = hash; + response.timestamp = response_time; + m_pending_chunk_hashes[key] = response; + } +} + +bool ChunkValidationManager::is_chunk_being_validated(uint32_t chunk_index) const +{ + std::lock_guard lock(m_chunk_validation_mutex); + return (m_current_validating_chunk_index == chunk_index); +} + +bool ChunkValidationManager::validate_unverified_chunks() +{ + // Check if we have unverified chunks first (fast check) + std::vector unverified_chunks = m_core.getCheckpointList().get_unverified_chunks(); + if (unverified_chunks.empty()) + { + return false; // No unverified chunks - nothing to validate + } + + // We need at least some peers to validate (but don't require full synchronization) + // Validation can happen during sync as long as we have peers + if (m_peersCount.load() == 0) + { + logger(DEBUGGING) << "Cannot validate chunks: no peers connected yet"; + return false; + } + + // Rate limit: don't attempt validation more than once every P2P_CHECKPOINT_LIST_RE_REQUEST seconds (5 minutes) + // EXCEPTION: Allow immediate attempt if this is the first time we have unverified chunks + // (bypass rate limit on first attempt after chunks are created) + uint64_t time_now = time(nullptr); + bool bypass_rate_limit = false; + { + std::lock_guard lock(m_chunk_validation_mutex); + + // Check if this is the first validation attempt (m_last_chunk_validation_attempt == 0) + // or if enough time has passed + uint64_t time_since_last = time_now - m_last_chunk_validation_attempt; + + if (m_last_chunk_validation_attempt == 0) + { + // First validation attempt - allow it immediately + bypass_rate_limit = true; + logger(INFO) << "First chunk validation attempt - bypassing rate limit"; + } + else if (time_since_last < cn::P2P_CHECKPOINT_LIST_RE_REQUEST) + { + // Too soon since last attempt + logger(DEBUGGING) << "Chunk validation rate limited: " + << (cn::P2P_CHECKPOINT_LIST_RE_REQUEST - time_since_last) + << " seconds remaining"; + return false; + } + + m_last_chunk_validation_attempt = time_now; + } + + // Sort to ensure chronological order (oldest first) + std::sort(unverified_chunks.begin(), unverified_chunks.end()); + + logger(INFO) << "Attempting to validate " << unverified_chunks.size() + << " unverified chunk(s) via P2P consensus"; + + // Check if we're already validating a chunk + { + std::lock_guard lock(m_chunk_validation_mutex); + if (m_current_validating_chunk_index != UINT32_MAX) + { + // Already validating a chunk - wait for it to complete + return false; + } + } + + // Get current blockchain height to calculate peer uptime in blocks + uint32_t current_height; + crypto::Hash blockId; + m_core.get_blockchain_top(current_height, blockId); + uint32_t block_time = m_currency.difficultyTarget(); // Block time in seconds (120 for mainnet/testnet) + uint32_t min_uptime_blocks = m_core.getCheckpointList().get_min_peer_uptime_blocks(); + + // Find eligible peers (meet uptime requirement and support chunk-based checkpoints) + std::vector eligible_peers; + std::map peer_network_16; // peer_id -> /16 network + + // First, log all connected peers for debugging + uint32_t total_connected = 0; + m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { + total_connected++; + }); + logger(INFO) << "Checking " << total_connected << " connected peer(s) for chunk validation eligibility"; + + m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { + // Only consider peers that support chunk-based checkpoints + if (ctx.version < cn::P2P_CHECKPOINT_LIST_VERSION) + { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: version " + << static_cast(ctx.version) << " < " << cn::P2P_CHECKPOINT_LIST_VERSION; + return; // Skip old version peers + } + + // Accept peers in normal state (synchronized) OR synchronizing state + // We can validate chunks even during sync, as long as peers are connected + if (ctx.m_state != CryptoNoteConnectionContext::state_normal && + ctx.m_state != CryptoNoteConnectionContext::state_synchronizing) + { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: state " + << get_protocol_state_string(ctx.m_state) << " (need normal or synchronizing)"; + return; // Skip peers that aren't in a usable state + } + + // Calculate peer uptime in blocks + // Uptime = (current_time - connection_start_time) / block_time + // This is an approximation - ideally we'd track peer's blockchain height when they first connected + time_t connection_duration = time_now - ctx.m_started; + if (connection_duration < 0) + { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: invalid connection time"; + return; // Invalid connection time + } + + uint32_t peer_uptime_blocks = static_cast(connection_duration / block_time); + + // Check if peer meets minimum uptime requirement + if (peer_uptime_blocks < min_uptime_blocks) + { + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: uptime " + << peer_uptime_blocks << " blocks < " << min_uptime_blocks << " blocks required"; + return; // Peer doesn't meet uptime requirement + } + + // Peer is eligible + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " is ELIGIBLE: version " + << static_cast(ctx.version) << ", state " + << get_protocol_state_string(ctx.m_state) << ", uptime " + << peer_uptime_blocks << " blocks"; + eligible_peers.push_back(peer_id); + peer_network_16[peer_id] = CheckpointList::get_network_16(ctx.m_remote_ip); + }); + + if (eligible_peers.empty()) + { + logger(INFO) << "No eligible peers for chunk validation (need uptime > " + << min_uptime_blocks << " blocks, version 2+, and in normal/synchronizing state)"; + logger(INFO) << "Note: Only ACTIVE CONNECTIONS are considered, not peers in peerlist. " + << "Use 'print_cn' command to see active connections. " + << "To force connection to a peer, use --add-priority-node :"; + logger(INFO) << "Chunk validation will be retried when eligible peers become available"; + return false; + } + + logger(INFO) << "Found " << eligible_peers.size() << " eligible peer(s) for chunk validation " + << "(uptime > " << min_uptime_blocks << " blocks, support version 2+)"; + + // Validate chunks in chronological order (oldest first) + // This ensures we catch divergences at the root cause + for (uint32_t chunk_index : unverified_chunks) + { + // Check if we should stop (node shutting down) + if (m_stop.load()) + { + break; + } + + // Mark this chunk as being validated + { + std::lock_guard lock(m_chunk_validation_mutex); + if (m_current_validating_chunk_index != UINT32_MAX) + { + // Another thread started validation + break; + } + m_current_validating_chunk_index = chunk_index; + } + + // Get local chunk hash + crypto::Hash local_chunk_hash = m_core.getCheckpointList().get_chunk_hash(chunk_index); + if (local_chunk_hash == NULL_HASH) + { + logger(WARNING) << "Cannot validate chunk " << chunk_index << ": chunk hash not found in memory"; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + + logger(INFO) << "Validating chunk " << chunk_index << " (oldest unverified chunk) " + << "with " << eligible_peers.size() << " eligible peer(s)"; + + // Check if this chunk is already being validated asynchronously + { + std::lock_guard lock(m_pending_validations_mutex); + if (m_pending_validations.find(chunk_index) != m_pending_validations.end()) + { + logger(DEBUGGING) << "Chunk " << chunk_index << " is already being validated asynchronously, skipping"; + { + std::lock_guard lock2(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + } + + // Calculate consensus requirements (M, K, n) + CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(eligible_peers.size()); + logger(INFO) << "Consensus requirements (testnet): M=" << req.min_agreements + << ", K=" << req.min_peers << ", n=" << req.min_diverse_networks + << " (have " << eligible_peers.size() << " available peers)"; + + // Sample K peers with network diversity using shared utility function + auto getPeerNetwork16 = [&peer_network_16](uint64_t peer_id) -> uint32_t { + auto it = peer_network_16.find(peer_id); + return (it != peer_network_16.end()) ? it->second : 0; + }; + + CheckpointList::PeerSamplingResult sampling_result = CheckpointList::sample_peers_with_diversity( + eligible_peers, + getPeerNetwork16, + req.min_peers); + + const std::vector& sampled_peers = sampling_result.sampled_peers; + const std::map& network_votes = sampling_result.network_votes; + + if (sampled_peers.empty()) + { + logger(WARNING) << "Could not sample any peers for chunk " << chunk_index << " validation"; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + + // Verify we have at least n distinct networks (network diversity requirement) + uint32_t distinct_networks = static_cast(network_votes.size()); + + if (distinct_networks < req.min_diverse_networks) + { + logger(WARNING) << "Could not achieve network diversity for chunk " << chunk_index + << " validation: have " << distinct_networks + << " distinct networks, need " << req.min_diverse_networks + << ". Sampled " << sampled_peers.size() << " peer(s)."; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + + logger(INFO) << "Sampled " << sampled_peers.size() << " peer(s) from " << distinct_networks + << " distinct network(s) (requirement: " << req.min_diverse_networks << " networks)"; + + // Send async requests to sampled peers + uint64_t request_time = time(nullptr); + uint32_t requests_sent = 0; + for (uint64_t peer_id : sampled_peers) + { + if (send_chunk_hash_request_async(peer_id, chunk_index)) + { + requests_sent++; + } + } + + if (requests_sent == 0) + { + logger(WARNING) << "Failed to send any async requests for chunk " << chunk_index; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + + // Store pending validation state + { + std::lock_guard lock(m_pending_validations_mutex); + PendingChunkValidation pending; + pending.chunk_index = chunk_index; + pending.request_timestamp = request_time; + pending.attempt_start_time = request_time; + pending.attempt_number = 1; + pending.requested_peers = sampled_peers; + pending.local_hash = local_chunk_hash; + pending.is_first_attempt = true; + m_pending_validations[chunk_index] = pending; + } + + logger(INFO) << "Sent async chunk hash requests for chunk " << chunk_index + << " to " << requests_sent << " peer(s). " + << "Will check for consensus after 2 minutes."; + + // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + + // Don't process next chunk until this one is validated (file must be sequential) + // The check_pending_chunk_validations() function will handle consensus checking + break; + } + + return true; +} + +void ChunkValidationManager::check_pending_chunk_validations() +{ + uint64_t time_now = time(nullptr); + const uint64_t CONSENSUS_WAIT_SECONDS = 120; // 2 minutes + const uint64_t RETRY_DELAY_SECONDS = 60; // 1 minute delay before retry + + std::lock_guard lock(m_pending_validations_mutex); + + // Iterate through pending validations + for (auto it = m_pending_validations.begin(); it != m_pending_validations.end();) + { + uint32_t chunk_index = it->first; + PendingChunkValidation& pending = it->second; + + uint64_t elapsed = time_now - pending.request_timestamp; + + // Check if 2 minutes have passed since requests were sent + if (elapsed < CONSENSUS_WAIT_SECONDS) + { + // Not enough time has passed yet, skip this validation + ++it; + continue; + } + + // 2 minutes have passed - check for consensus + logger(INFO) << "Checking consensus for chunk " << chunk_index + << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; + + // Collect responses from requested peers + std::unordered_map> hash_votes; // hash -> vote count + uint32_t agreements = 0; + uint32_t null_hash_responses = 0; + uint32_t responses_received = 0; + + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + + for (uint64_t peer_id : pending.requested_peers) + { + auto response_it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); + if (response_it != m_pending_chunk_hashes.end()) + { + crypto::Hash peer_hash = response_it->second.hash; + responses_received++; + + if (peer_hash == NULL_HASH) + { + null_hash_responses++; + continue; + } + + // Count votes for this hash + hash_votes[peer_hash]++; + + if (peer_hash == pending.local_hash) + { + agreements++; + } + } + } + } + + // Calculate consensus requirements + CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(pending.requested_peers.size()); + + logger(INFO) << "Chunk " << chunk_index << " consensus check: received " << responses_received + << " response(s) from " << pending.requested_peers.size() << " requested peer(s). " + << "Agreements: " << agreements << " (need M=" << req.min_agreements << "), " + << "NULL_HASH responses: " << null_hash_responses; + + // Check if we have M agreements (consensus reached) + if (agreements >= req.min_agreements) + { + // Consensus reached - save to checkpoint.dat + logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index + << " validated via peer consensus (" << agreements + << " agreements, need M=" << req.min_agreements << ")"; + + if (m_core.getCheckpointList().add_verified_chunk_to_file(chunk_index)) + { + logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index + << " saved to checkpoint.dat"; + + // Remove pending validation + it = m_pending_validations.erase(it); + + // Clean up responses for this chunk + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } + continue; + } + else + { + logger(ERROR) << "Failed to save validated chunk " << chunk_index << " to checkpoint.dat"; + // Remove pending validation anyway (we'll retry later) + it = m_pending_validations.erase(it); + continue; + } + } + + // Check if we have M peers agreeing on a DIFFERENT hash (actual divergence) + crypto::Hash consensus_hash = NULL_HASH; + uint32_t max_votes = 0; + for (const auto& vote : hash_votes) + { + if (vote.first != pending.local_hash && vote.second > max_votes) + { + max_votes = vote.second; + consensus_hash = vote.first; + } + } + + bool has_divergence = (consensus_hash != NULL_HASH && max_votes >= req.min_agreements); + + // If all responses are NULL_HASH or missing, peers don't have this chunk yet (not a divergence) + if (responses_received == 0 || (responses_received == null_hash_responses && !has_divergence)) + { + logger(INFO) << "Chunk " << chunk_index + << " validation: No peers have this chunk in memory yet (all returned NULL_HASH or no response). " + << "This is normal if: (1) peers are using version 1 (don't support chunk checkpoints), " + << "or (2) peers haven't created this chunk yet. " + << "Will retry validation once peers create this chunk."; + + // Remove pending validation - we'll retry later when peers have the chunk + it = m_pending_validations.erase(it); + + // Clean up responses + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } + continue; + } + + // Consensus not reached - check if we should retry + if (pending.is_first_attempt) + { + // First attempt failed - wait 1 more minute (3 minutes total) before retry + uint64_t total_elapsed = time_now - pending.attempt_start_time; + if (total_elapsed < (CONSENSUS_WAIT_SECONDS + RETRY_DELAY_SECONDS)) + { + // Still waiting for retry delay + ++it; + continue; + } + + // Retry delay passed - start second attempt + logger(INFO) << "Chunk " << chunk_index + << " consensus failed on first attempt. Starting second attempt..."; + + // Send new requests to same peers (or get new eligible peers) + // For now, reuse same peers + uint64_t retry_time = time_now; + uint32_t requests_sent = 0; + for (uint64_t peer_id : pending.requested_peers) + { + if (send_chunk_hash_request_async(peer_id, chunk_index)) + { + requests_sent++; + } + } + + if (requests_sent > 0) + { + // Update pending validation for second attempt + pending.request_timestamp = retry_time; + pending.attempt_number = 2; + pending.is_first_attempt = false; + + logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index + << " to " << requests_sent << " peer(s). " + << "Will check for consensus after 2 minutes."; + } + else + { + logger(WARNING) << "Failed to send second attempt requests for chunk " << chunk_index; + // Remove pending validation + it = m_pending_validations.erase(it); + continue; + } + } + else + { + // Second attempt also failed - check if it's actual divergence or just no responses + + // Check if we have M peers agreeing on a DIFFERENT hash (actual divergence) + crypto::Hash consensus_hash_second = NULL_HASH; + uint32_t max_votes_second = 0; + for (const auto& vote : hash_votes) + { + if (vote.first != pending.local_hash && vote.second > max_votes_second) + { + max_votes_second = vote.second; + consensus_hash_second = vote.first; + } + } + + bool has_divergence_second = (consensus_hash_second != NULL_HASH && max_votes_second >= req.min_agreements); + + // If all responses are NULL_HASH or missing, peers don't have this chunk yet (not a divergence) + if (responses_received == 0 || (responses_received == null_hash_responses && !has_divergence_second)) + { + logger(INFO) << "Chunk " << chunk_index + << " validation: No peers have this chunk in memory yet after 2 attempts " + << "(all returned NULL_HASH or no response). " + << "This is normal if: (1) peers are using version 1 (don't support chunk checkpoints), " + << "or (2) peers haven't created this chunk yet. " + << "Will retry validation once peers create this chunk."; + + // Remove pending validation - we'll retry later when peers have the chunk + it = m_pending_validations.erase(it); + + // Clean up responses + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } + continue; + } + + // Actual divergence: M peers agree on different hash + if (has_divergence_second) + { + logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index + << " validation FAILED: peer consensus did not agree with local chunk hash " + << "after 2 attempts. M peers (" << max_votes_second + << ") agree on different hash: " << consensus_hash_second + << " (local: " << pending.local_hash << "). " + << "This indicates blockchain divergence."; + + // TODO: Handle rollback (same logic as before) + // For now, just remove pending validation + it = m_pending_validations.erase(it); + + // Clean up responses + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } + continue; + } + + // Mixed responses (some NULL_HASH, some mismatches, but not M agreements on different hash) + logger(WARNING) << "Chunk " << chunk_index + << " validation: Inconsistent results after 2 attempts. " + << "Some peers returned NULL_HASH, some returned different hashes, " + << "but no M peers agreed on a single different hash. " + << "This may indicate network issues or peers still syncing. " + << "Will retry validation later."; + + // Remove pending validation - we'll retry later + it = m_pending_validations.erase(it); + + // Clean up responses + { + std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } + } + continue; + } + + ++it; + } +} + +} // namespace cn + diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h new file mode 100644 index 000000000..1d4e24bde --- /dev/null +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h @@ -0,0 +1,105 @@ +// Copyright (c) 2011-2017 The Cryptonote developers +// Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs +// Copyright (c) 2018-2023 Conceal Network & Conceal Devs +// +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#pragma once + +#include +#include +#include +#include +#include + +#include "CryptoNoteCore/CryptoNoteBasic.h" +#include "CryptoNoteCore/CheckpointList.h" +#include "CryptoNoteCore/Currency.h" +#include "CryptoNoteCore/ICore.h" +#include "P2p/NetNodeCommon.h" +#include + +namespace cn +{ + + /** + * ChunkValidationManager + * + * Handles asynchronous chunk validation via P2P consensus. + * Manages pending chunk hash requests, responses, and consensus checking. + */ + class ChunkValidationManager + { + public: + ChunkValidationManager(ICore& core, + IP2pEndpoint* p2p, + const Currency& currency, + logging::ILogger& log, + std::atomic& peersCount, + std::atomic& stop); + + // Validate unverified chunks in chronological order (oldest first) + // This ensures we catch divergences at the root cause and rollback to the correct point + // Only validates when peers meet uptime requirements + // @return true if validation was attempted (even if it failed) + bool validate_unverified_chunks(); + + // Check pending chunk validations for consensus (called periodically) + // Checks if we have M identical hashes within the time window (2 minutes) + // If consensus reached, processes it; if timeout, schedules retry + void check_pending_chunk_validations(); + + // Send chunk hash request asynchronously (non-blocking) + // Returns true if request was sent successfully + bool send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index); + + // Store chunk hash response (called from message handler) + void store_chunk_hash_response(uint64_t peer_id, uint32_t chunk_index, const crypto::Hash& hash); + + // Check if chunk is currently being validated + bool is_chunk_being_validated(uint32_t chunk_index) const; + + private: + // Pending chunk hash responses: (peer_id, chunk_index) -> (hash, timestamp) + // Used for asynchronous consensus mechanism + struct ChunkHashResponse { + crypto::Hash hash; + uint64_t timestamp; // When response was received + }; + + // Pending chunk validation attempts: chunk_index -> validation state + struct PendingChunkValidation { + uint32_t chunk_index; + uint64_t request_timestamp; // When requests were sent + uint64_t attempt_start_time; // When this attempt started + uint32_t attempt_number; // 1 or 2 + std::vector requested_peers; // Peers we requested from + crypto::Hash local_hash; + bool is_first_attempt; + }; + + ICore& m_core; + IP2pEndpoint* m_p2p; + const Currency& m_currency; + logging::LoggerRef logger; + std::atomic& m_peersCount; + std::atomic& m_stop; + + // Pending chunk hash responses: (peer_id, chunk_index) -> (hash, timestamp) + mutable std::mutex m_pending_chunk_hashes_mutex; + std::map, ChunkHashResponse> m_pending_chunk_hashes; + + // Pending chunk validation attempts: chunk_index -> validation state + mutable std::mutex m_pending_validations_mutex; + std::map m_pending_validations; // chunk_index -> validation state + + // Chunk validation state: track which chunk we're currently validating + // This prevents duplicate validation attempts and ensures chronological order + mutable std::mutex m_chunk_validation_mutex; + uint32_t m_current_validating_chunk_index; // Currently validating chunk (or UINT32_MAX if none) + uint64_t m_last_chunk_validation_attempt; // Last time we attempted validation (to avoid spamming) + }; + +} // namespace cn + From 102b979d3607d3595952758058bfc688e8788212 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 1 Dec 2025 17:15:23 -0500 Subject: [PATCH 16/56] fix p2p pointer --- .../CryptoNoteProtocolHandler.cpp | 6 ++++++ .../CryptoNoteProtocolHandlerChunk.cpp | 13 +++++++++++++ .../CryptoNoteProtocolHandlerChunk.h | 3 +++ 3 files changed, 22 insertions(+) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 0564ace39..2cc603ff1 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -90,6 +90,12 @@ void CryptoNoteProtocolHandler::set_p2p_endpoint(IP2pEndpoint *p2p) m_p2p = p2p; else m_p2p = &m_p2p_stub; + + // Update chunk validation manager with the new P2P endpoint + if (m_chunkValidationManager) + { + m_chunkValidationManager->set_p2p_endpoint(m_p2p); + } } void CryptoNoteProtocolHandler::onConnectionOpened(CryptoNoteConnectionContext &context) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index befc0de28..6d637b91e 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -57,6 +57,19 @@ ChunkValidationManager::ChunkValidationManager(ICore& core, { } +void ChunkValidationManager::set_p2p_endpoint(IP2pEndpoint* p2p) +{ + if (p2p) + { + m_p2p = p2p; + logger(INFO) << "Updated P2P endpoint in ChunkValidationManager"; + } + else + { + logger(WARNING) << "Attempted to set null P2P endpoint in ChunkValidationManager"; + } +} + bool ChunkValidationManager::send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index) { // Find the peer connection diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h index 1d4e24bde..f317c1a9b 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h @@ -39,6 +39,9 @@ namespace cn std::atomic& peersCount, std::atomic& stop); + // Update P2P endpoint (called when endpoint is set/updated) + void set_p2p_endpoint(IP2pEndpoint* p2p); + // Validate unverified chunks in chronological order (oldest first) // This ensures we catch divergences at the root cause and rollback to the correct point // Only validates when peers meet uptime requirements From 1f6983084b8e7f32ed90819bd49dde74a2cebaa7 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 15 Dec 2025 18:24:35 -0500 Subject: [PATCH 17/56] test chechcpoint * combo config + dns + p2p --- src/CryptoNoteConfig.h | 15 +++- src/CryptoNoteCore/Blockchain.cpp | 55 ++++++++------ src/CryptoNoteCore/Blockchain.h | 3 + src/CryptoNoteCore/CheckpointsList.cpp | 73 +++++++++++++------ src/CryptoNoteCore/Core.h | 2 +- src/CryptoNoteCore/ICore.h | 1 + .../CryptoNoteProtocolHandler.cpp | 6 +- .../CryptoNoteProtocolHandlerChunk.cpp | 32 +++++++- 8 files changed, 132 insertions(+), 55 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index 45b17ceb1..ce7f8fc3e 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -254,7 +254,7 @@ namespace cn #endif const char DNS_CHECKPOINT_DOMAIN[] = "checkpoints.conceal.id"; - const char TESTNET_DNS_CHECKPOINT_DOMAIN[] = "testpoints.conceal.gq"; + const char TESTNET_DNS_CHECKPOINT_DOMAIN[] = "testpoints.conceal.id"; // Blockchain Checkpoints: // {, ""}, @@ -447,7 +447,15 @@ namespace cn {1840000, "b3a95fbf906555264a31906678b299c0851d09628b8d37969a285cb54a614c31"}, {1850000, "041d84a8c1fb68d24e6a9d8fe5aab27c6db654ecaad3b2d541c40d6e9df53feb"}, {1860000, "63642037f4d5d82150c120e776a3b2d2a80f725b4759ffda12c94bd45499fb25"}, - {1870000, "159fa13f6f9f48e2d8344e76ebdfe2ea986b9a72e94325dacbbd3bd73cbdbb41"} + {1870000, "159fa13f6f9f48e2d8344e76ebdfe2ea986b9a72e94325dacbbd3bd73cbdbb41"}, + {1880000, "7e5dd6c104989975b482d7bf277934d8f9252e81f5bc996ed7c54ee95542ade6"}, + {1890000, "c2ded97cfaefc52aa1de9f6ca8f04d2f3c2ab3b48bf572801555e9fa7934f954"}, + {1900000, "50f2fc1af569f1ebb6567b915e2ee1bf72c585f544725e952a303b447faab909"}, + {1910000, "e2f3e1560369c4e457fdeffbe2ca2fbf9db7aa89a5cd6d9d2e96f7ef826690eb"}, + {1920000, "ba318eb902d6a3f7dd4e7116b13c6b1f2cb4e8c9e70acbde3b6a28a1374c6243"}, + {1930000, "6f39f0888c14808710e909fb45c864fdf738cf644e432486d8e01b65dbbc2862"}, + {1940000, "fe774dd97a5b975c02abb18ca48bf6fd3a8048b597520befe1f261855a7ccf3f"}, + {1950000, "6147f276271c55319cf6fc03e86bcaa75ea0a2aab126095a9a9f8a811eb229e1"} }; const std::initializer_list TESTNET_CHECKPOINTS = { @@ -489,7 +497,8 @@ namespace cn {875000, "91eac54f608c1d0e43111c107a2f5caf259bcbbd66714b6591cedbc64f9c4cdf"}, {900000, "a70b6df1794a6d91071cd5fc87719769bf09610d520c2c2134f53908d1e3de40"}, {925000, "a00b47f3610cfd5c509183322fca89e388c0427601ce4db7e397acbcab5a3ee6"}, - {950000, "387573b7b9bdbc1d79c28156cf15d7e08ddf248a0257b0ee7ef2731c5c7a0534"} + {950000, "387573b7b9bdbc1d79c28156cf15d7e08ddf248a0257b0ee7ef2731c5c7a0534"}, + {975000, "2bbf6d2fecb329d9c34968e60b0d2814a2d5a2f69ee872b77d152ba795e881ae"} }; } // namespace cn diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 845bfd218..40e4e5a61 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -3764,28 +3764,7 @@ namespace cn try { std::lock_guard lk(m_blockchain_lock); - - logger(INFO, BRIGHT_YELLOW) << "Rolling back blockchain to height " << height; - - // Check if we're already at or below the requested height - if (height >= m_blocks.size()) - { - logger(WARNING, BRIGHT_YELLOW) << "Requested rollback to height " << height - << " which is higher than current height " << m_blocks.size(); - return true; - } - - while (m_blocks.size() > height + 1) - { - if (!removeLastBlock()) - { - logger(ERROR, BRIGHT_RED) << "Failed to remove last block during rollback"; - return false; - } - } - - logger(INFO, BRIGHT_GREEN) << "Blockchain successfully rolled back to height: " << height << "Synchronization will resume"; - return true; + return rollback_to_height_impl(height); } catch (const std::exception&) { @@ -3794,6 +3773,38 @@ namespace cn } } + bool Blockchain::rollback_to_height_impl(uint32_t height) + { + logger(INFO, BRIGHT_YELLOW) << "Rolling back blockchain to height " << height; + + // Check if we're already at or below the requested height + if (height >= m_blocks.size()) + { + logger(WARNING, BRIGHT_YELLOW) << "Requested rollback to height " << height + << " which is higher than current height " << m_blocks.size(); + return true; + } + + // Validate height is reasonable (not 0, which would rollback genesis) + if (height == 0 && m_blocks.size() > 1) + { + logger(ERROR, BRIGHT_RED) << "Cannot rollback to height 0 (genesis block) - current height: " << m_blocks.size(); + return false; + } + + while (m_blocks.size() > height + 1) + { + if (!removeLastBlock()) + { + logger(ERROR, BRIGHT_RED) << "Failed to remove last block during rollback"; + return false; + } + } + + logger(INFO, BRIGHT_GREEN) << "Blockchain successfully rolled back to height: " << height << "Synchronization will resume"; + return true; + } + bool Blockchain::removeLastBlock() { if (m_blocks.empty()) diff --git a/src/CryptoNoteCore/Blockchain.h b/src/CryptoNoteCore/Blockchain.h index ee3589792..fba03e42d 100644 --- a/src/CryptoNoteCore/Blockchain.h +++ b/src/CryptoNoteCore/Blockchain.h @@ -225,6 +225,9 @@ namespace cn bool have_tx_keyimg_as_spent(const crypto::KeyImage &key_im); private: + // Private helper for rollback - performs the actual rollback operation + // Precondition: m_blockchain_lock must be held + bool rollback_to_height_impl(uint32_t height); bool m_testnet = false; struct MultisignatureOutputUsage { diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 163a3c4bc..10e00dfa7 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -154,27 +154,45 @@ namespace cn { uint32_t dns_checkpoints_invalid = 0; for (const auto& record : records) { - uint32_t height; - crypto::Hash hash = NULL_HASH; - std::stringstream ss; - size_t del = record.find_first_of(':'); + // Support both formats: + // 1. One checkpoint per record: "height:hash" + // 2. Multiple checkpoints per record (newline-separated): "height1:hash1\nheight2:hash2\n..." + + std::stringstream record_stream(record); + std::string line; + bool record_has_valid_checkpoint = false; + + while (std::getline(record_stream, line)) { + // Skip empty lines + if (line.empty() || (line.find_first_not_of(" \t\r\n") == std::string::npos)) { + continue; + } + // Trim whitespace + line.erase(0, line.find_first_not_of(" \t")); + line.erase(line.find_last_not_of(" \t") + 1); + + uint32_t height; + crypto::Hash hash = NULL_HASH; + std::stringstream ss; + size_t del = line.find_first_of(':'); + if (del == std::string::npos) { - logger(DEBUGGING) << "Invalid DNS checkpoint record format (missing ':'): " << record; + logger(DEBUGGING) << "Invalid DNS checkpoint format (missing ':'): " << line; dns_checkpoints_invalid++; continue; } - std::string height_str = record.substr(0, del), hash_str = record.substr(del + 1, 64); - ss.str(height_str); - ss >> height; - char c; - - if ((ss.fail() || ss.get(c)) || !common::podFromHex(hash_str, hash)) { - logger(DEBUGGING) << "Failed to parse DNS checkpoint record: " << record; + std::string height_str = line.substr(0, del), hash_str = line.substr(del + 1, 64); + ss.str(height_str); + ss >> height; + char c; + + if ((ss.fail() || ss.get(c)) || !common::podFromHex(hash_str, hash)) { + logger(DEBUGGING) << "Failed to parse DNS checkpoint: " << line; dns_checkpoints_invalid++; - continue; - } + continue; + } // Check if this height already exists in CryptoNoteConfig.h (PRIORITY 1 takes precedence) if (m_old_checkpoint_hashes.count(height) > 0) { @@ -189,13 +207,20 @@ namespace cn { // Also add as target for validation (if not already a target) if (m_targets.count(height) == 0) { - add_checkpoint_target(height, hash_str); + add_checkpoint_target(height, hash_str); } dns_checkpoints_added++; + record_has_valid_checkpoint = true; logger(INFO) << "Added DNS checkpoint (PRIORITY 2): " << height_str << ":" << hash_str; } + // If a record had no valid checkpoints, count it as invalid + if (!record_has_valid_checkpoint && !record.empty()) { + dns_checkpoints_invalid++; + } + } + if (dns_checkpoints_added > 0) { logger(INFO) << "Stored " << m_dns_checkpoint_hashes.size() @@ -292,10 +317,11 @@ namespace cn { * * CHUNKING EXPLANATION: * Instead of storing all 1.87M block hashes (60MB), we store chunk hashes: - * - Chunk 0: hash(blocks 0-9999) -> 1 hash (32 bytes) - * - Chunk 1: hash(blocks 10000-19999) -> 1 hash (32 bytes) + * NOTE: Block 0 (genesis) is EXCLUDED from chunks + * - Chunk 0: hash(blocks 1-10000) -> 1 hash (32 bytes) + * - Chunk 1: hash(blocks 10001-20000) -> 1 hash (32 bytes) * - ... - * - Chunk 186: hash(blocks 1860000-1869999) -> 1 hash (32 bytes) + * - Chunk 186: hash(blocks 1860001-1870000) -> 1 hash (32 bytes) * * Total: 187 chunks × 32 bytes = ~6KB (vs 60MB for full list) * @@ -333,7 +359,6 @@ namespace cn { const std::lock_guard lock(m_chunks_lock); // Calculate how many chunks we need - // SIMPLIFIED CHUNK STRUCTURE (block 0/genesis is NOT included in chunks): // chunk[0]: blocks 1 to chunk_size (inclusive) = chunk_size blocks // chunk[1]: blocks (chunk_size + 1) to (2 * chunk_size) (inclusive) = chunk_size blocks // chunk[n]: blocks (n * chunk_size + 1) to ((n + 1) * chunk_size) (inclusive) = chunk_size blocks @@ -358,7 +383,6 @@ namespace cn { // Generate each chunk for (uint32_t chunk_index = 0; chunk_index < num_chunks; chunk_index++) { - // SIMPLIFIED: All chunks are uniform (chunk_size blocks each) // chunk[0]: blocks 1 to chunk_size // chunk[1]: blocks (chunk_size + 1) to (2 * chunk_size) // chunk[n]: blocks (n * chunk_size + 1) to ((n + 1) * chunk_size) @@ -398,12 +422,12 @@ namespace cn { { std::stringstream ss; ss << "Validated and replaced " << checkpoint_result.checkpoints_in_chunk.size() << " checkpoint(s) in chunk " << chunk_index - << " (heights: "; + << " (blocks " << chunk_start_height << "-" << chunk_end_height << ", heights: "; for (size_t i = 0; i < checkpoint_result.checkpoints_in_chunk.size(); i++) { if (i > 0) ss << ", "; ss << checkpoint_result.checkpoints_in_chunk[i]; } - ss << ") - Priority: " << checkpoint_result.checkpoints_from_config << " from CryptoNoteConfig.h, " + ss << ") - " << checkpoint_result.checkpoints_from_config << " from CryptoNoteConfig.h, " << checkpoint_result.checkpoints_from_dns << " from DNS, rest from blockchain.dat"; logger(INFO) << ss.str(); } @@ -513,8 +537,9 @@ namespace cn { { logger(INFO) << "Applied " << total_checkpoints_applied << " checkpoint(s) to chunk " << chunk_index - << " (Priority: " << checkpoint_result.checkpoints_from_config << " from CryptoNoteConfig.h, " - << checkpoint_result.checkpoints_from_dns << " from DNS, rest from blockchain.dat)"; + << " (blocks " << chunk_start_height << "-" << chunk_end_height << "): " + << checkpoint_result.checkpoints_from_config << " from CryptoNoteConfig.h, " + << checkpoint_result.checkpoints_from_dns << " from DNS, rest from blockchain.dat"; } // Compute hash of this chunk's block IDs diff --git a/src/CryptoNoteCore/Core.h b/src/CryptoNoteCore/Core.h index 193e1a84c..40a619b4d 100644 --- a/src/CryptoNoteCore/Core.h +++ b/src/CryptoNoteCore/Core.h @@ -97,7 +97,7 @@ namespace cn { virtual void get_blockchain_top(uint32_t& height, crypto::Hash& top_id) override; bool get_blocks(uint32_t start_offset, uint32_t count, std::list& blocks, std::list& txs); bool get_blocks(uint32_t start_offset, uint32_t count, std::list& blocks); - bool rollback_chain_to(uint32_t height); + bool rollback_chain_to(uint32_t height) override; crypto::Hash checkpoint_hash(uint32_t height); template bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) diff --git a/src/CryptoNoteCore/ICore.h b/src/CryptoNoteCore/ICore.h index faeb5ca5a..c6ee56436 100644 --- a/src/CryptoNoteCore/ICore.h +++ b/src/CryptoNoteCore/ICore.h @@ -117,6 +117,7 @@ class ICore { virtual bool addMessageQueue(MessageQueue& messageQueue) = 0; virtual bool removeMessageQueue(MessageQueue& messageQueue) = 0; + virtual bool rollback_chain_to(uint32_t height) = 0; }; } //namespace cn diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 2cc603ff1..0cc731921 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -471,14 +471,14 @@ int CryptoNoteProtocolHandler::handle_request_get_objects(int command, NOTIFY_RE size_t maxObjects = m_maxObjectCount.load(); const size_t totalObjects = arg.blocks.size() + arg.txs.size(); - logger(INFO) << "DEBUG: Request for " << totalObjects << " objects (limit: " << maxObjects << ")"; - - if (totalObjects > maxObjects) { + std::string ipAddress = common::ipAddressToString(context.m_remote_ip); logger(logging::ERROR) << context << "Requested objects count exceeds limit of " << maxObjects << ": blocks " << arg.blocks.size() << " + txs " << arg.txs.size() << " = " << totalObjects; + logger(logging::WARNING) << "IP " << ipAddress << ":" << context.m_remote_port + << " attempted to overload with " << totalObjects << " objects (limit: " << maxObjects << ")"; context.m_state = CryptoNoteConnectionContext::state_shutdown; return 1; } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index 6d637b91e..29504020d 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -659,8 +659,36 @@ void ChunkValidationManager::check_pending_chunk_validations() << " (local: " << pending.local_hash << "). " << "This indicates blockchain divergence."; - // TODO: Handle rollback (same logic as before) - // For now, just remove pending validation + // Calculate rollback height (bottom of the chunk) + uint32_t chunk_size = m_core.getCheckpointList().get_chunk_size(); + uint32_t rollback_height = chunk_index * chunk_size; + + logger(ERROR, BRIGHT_RED) << "Rolling back blockchain to height " << rollback_height + << " (chunk " << chunk_index << " boundary) due to peer consensus divergence"; + + // Truncate checkpoint.dat to the previous chunk (chunk_index - 1) + uint32_t last_valid_chunk = (chunk_index > 0) ? (chunk_index - 1) : 0; + if (!m_core.getCheckpointList().truncate_checkpoint_file(last_valid_chunk)) + { + logger(ERROR, BRIGHT_RED) << "Failed to truncate checkpoint.dat to chunk " << last_valid_chunk; + } + else + { + logger(INFO) << "Truncated checkpoint.dat to chunk " << last_valid_chunk; + } + + // Rollback blockchain to the chunk boundary + if (!m_core.rollback_chain_to(rollback_height)) + { + logger(ERROR, BRIGHT_RED) << "Failed to rollback blockchain to height " << rollback_height + << " - node may be in inconsistent state"; + } + else + { + logger(INFO, BRIGHT_GREEN) << "Successfully rolled back blockchain to height " << rollback_height; + } + + // Remove pending validation it = m_pending_validations.erase(it); // Clean up responses From 018b6f247e8d14d94da5b43b9df250225f797bb6 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 16 Dec 2025 07:35:12 -0500 Subject: [PATCH 18/56] invalid vs redundant --- src/CryptoNoteCore/Blockchain.cpp | 13 ++++++++++--- src/CryptoNoteCore/CheckpointsList.cpp | 12 ++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 40e4e5a61..38a2eb26f 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -914,9 +914,16 @@ namespace cn // If we're past the last chunk boundary, create missing chunks if (current_chunk_index > last_chunk_index && currentHeight > greatestTargetHeight) { - logger(INFO) << "Creating missing checkpoint chunks beyond last hardcoded checkpoint (from chunk " - << (last_chunk_index + 1) << " to " << current_chunk_index - << ", heights " << (currentCoveredHeight + 1) << " to " << currentHeight << ")"; + uint32_t num_chunks_to_create = current_chunk_index - last_chunk_index; + if (num_chunks_to_create == 1) { + logger(INFO) << "Creating missing checkpoint chunk " << current_chunk_index + << " beyond last hardcoded checkpoint (heights " + << (currentCoveredHeight + 1) << " to " << currentHeight << ")"; + } else { + logger(INFO) << "Creating missing checkpoint chunks beyond last hardcoded checkpoint (from chunk " + << (last_chunk_index + 1) << " to " << current_chunk_index + << ", heights " << (currentCoveredHeight + 1) << " to " << currentHeight << ")"; + } auto getBlockIdsFunc = [this](uint32_t startHeight, uint32_t maxCount) -> std::vector { return m_blockIndex.getBlockIds(startHeight, maxCount); diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 10e00dfa7..3fa6a8bad 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -161,6 +161,8 @@ namespace cn { std::stringstream record_stream(record); std::string line; bool record_has_valid_checkpoint = false; + bool record_has_skipped_checkpoint = false; + uint32_t invalid_lines_in_record = 0; while (std::getline(record_stream, line)) { // Skip empty lines @@ -179,7 +181,7 @@ namespace cn { if (del == std::string::npos) { logger(DEBUGGING) << "Invalid DNS checkpoint format (missing ':'): " << line; - dns_checkpoints_invalid++; + invalid_lines_in_record++; continue; } @@ -190,7 +192,7 @@ namespace cn { if ((ss.fail() || ss.get(c)) || !common::podFromHex(hash_str, hash)) { logger(DEBUGGING) << "Failed to parse DNS checkpoint: " << line; - dns_checkpoints_invalid++; + invalid_lines_in_record++; continue; } @@ -199,6 +201,7 @@ namespace cn { logger(DEBUGGING) << "Checkpoint at height " << height << " already exists in CryptoNoteConfig.h. " << "Skipping DNS checkpoint (CryptoNoteConfig.h takes precedence)."; dns_checkpoints_skipped++; + record_has_skipped_checkpoint = true; continue; } @@ -215,8 +218,9 @@ namespace cn { logger(INFO) << "Added DNS checkpoint (PRIORITY 2): " << height_str << ":" << hash_str; } - // If a record had no valid checkpoints, count it as invalid - if (!record_has_valid_checkpoint && !record.empty()) { + // Count this record as invalid ONLY if it had invalid lines AND no valid/skipped checkpoints + // (A record with only skipped checkpoints is valid but redundant, not invalid) + if (invalid_lines_in_record > 0 && !record_has_valid_checkpoint && !record_has_skipped_checkpoint && !record.empty()) { dns_checkpoints_invalid++; } } From 274b9c7585bc18e2344ca57ccd51b6e4deff32b9 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 16 Dec 2025 08:22:35 -0500 Subject: [PATCH 19/56] fix logging --- src/CryptoNoteCore/Blockchain.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 38a2eb26f..3283186b0 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -731,9 +731,9 @@ namespace cn } else { - logger(DEBUGGING) << "Skipping chunk " << chunk_idx - << " - not enough blocks (have " << currentHeight - << ", need " << chunk_end_height << ")"; + logger(INFO) << "Skipping chunk " << chunk_idx + << " - not enough blocks (have " << currentHeight + << ", need " << chunk_end_height << ")"; break; // Can't create more chunks if we don't have enough blocks } } @@ -808,9 +808,9 @@ namespace cn } else { - logger(DEBUGGING) << "Skipping chunk " << chunk_idx - << " - not enough blocks (have " << currentHeight - << ", need " << chunk_end_height << ")"; + logger(INFO) << "Skipping chunk " << chunk_idx + << " - not enough blocks (have " << currentHeight + << ", need " << chunk_end_height << ")"; break; } } @@ -916,11 +916,11 @@ namespace cn { uint32_t num_chunks_to_create = current_chunk_index - last_chunk_index; if (num_chunks_to_create == 1) { - logger(INFO) << "Creating missing checkpoint chunk " << current_chunk_index + logger(INFO) << "Checking for missing checkpoint chunk " << current_chunk_index << " beyond last hardcoded checkpoint (heights " << (currentCoveredHeight + 1) << " to " << currentHeight << ")"; } else { - logger(INFO) << "Creating missing checkpoint chunks beyond last hardcoded checkpoint (from chunk " + logger(INFO) << "Checking for missing checkpoint chunks beyond last hardcoded checkpoint (from chunk " << (last_chunk_index + 1) << " to " << current_chunk_index << ", heights " << (currentCoveredHeight + 1) << " to " << currentHeight << ")"; } @@ -964,9 +964,9 @@ namespace cn } else { - logger(DEBUGGING) << "Skipping chunk " << chunk_idx - << " - not enough blocks (have " << currentHeight - << ", need " << chunk_end_height << ")"; + logger(INFO) << "Skipping chunk " << chunk_idx + << " - not enough blocks (have " << currentHeight + << ", need " << chunk_end_height << ")"; break; } } @@ -1061,9 +1061,9 @@ namespace cn } else { - logger(DEBUGGING) << "Skipping chunk " << chunk_idx - << " - not enough blocks yet (have " << currentHeight - << ", need " << chunk_end_height << ")"; + logger(INFO) << "Skipping chunk " << chunk_idx + << " - not enough blocks yet (have " << currentHeight + << ", need " << chunk_end_height << ")"; break; // Can't create more chunks if we don't have enough blocks } } From fe2c34629479c63b7726c29003ace0177d0f8924 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 16 Dec 2025 09:13:08 -0500 Subject: [PATCH 20/56] filtering peers state --- .../CryptoNoteProtocolHandlerChunk.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index 29504020d..d376cabfa 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -208,18 +208,20 @@ bool ChunkValidationManager::validate_unverified_chunks() // Only consider peers that support chunk-based checkpoints if (ctx.version < cn::P2P_CHECKPOINT_LIST_VERSION) { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: version " - << static_cast(ctx.version) << " < " << cn::P2P_CHECKPOINT_LIST_VERSION; + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: P2P version " + << static_cast(ctx.version) << " < " << cn::P2P_CHECKPOINT_LIST_VERSION + << " (does not support chunk-based checkpoints)"; return; // Skip old version peers } - // Accept peers in normal state (synchronized) OR synchronizing state + // Accept peers in normal state (synchronized), idle state, OR synchronizing state // We can validate chunks even during sync, as long as peers are connected if (ctx.m_state != CryptoNoteConnectionContext::state_normal && + ctx.m_state != CryptoNoteConnectionContext::state_idle && ctx.m_state != CryptoNoteConnectionContext::state_synchronizing) { logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: state " - << get_protocol_state_string(ctx.m_state) << " (need normal or synchronizing)"; + << get_protocol_state_string(ctx.m_state) << " (need normal, idle, or synchronizing)"; return; // Skip peers that aren't in a usable state } @@ -244,7 +246,7 @@ bool ChunkValidationManager::validate_unverified_chunks() } // Peer is eligible - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " is ELIGIBLE: version " + logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " is ELIGIBLE for chunk validation: P2P version " << static_cast(ctx.version) << ", state " << get_protocol_state_string(ctx.m_state) << ", uptime " << peer_uptime_blocks << " blocks"; @@ -255,7 +257,7 @@ bool ChunkValidationManager::validate_unverified_chunks() if (eligible_peers.empty()) { logger(INFO) << "No eligible peers for chunk validation (need uptime > " - << min_uptime_blocks << " blocks, version 2+, and in normal/synchronizing state)"; + << min_uptime_blocks << " blocks, version 2+, and in normal/idle/synchronizing state)"; logger(INFO) << "Note: Only ACTIVE CONNECTIONS are considered, not peers in peerlist. " << "Use 'print_cn' command to see active connections. " << "To force connection to a peer, use --add-priority-node :"; From d592f56aa7d6f27929795f5f7b12fce49accf071 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 16 Dec 2025 12:16:07 -0500 Subject: [PATCH 21/56] logging debug K peer select --- src/CryptoNoteConfig.h | 2 +- .../CryptoNoteProtocolHandlerChunk.cpp | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index ce7f8fc3e..3535e2fc5 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -222,7 +222,7 @@ namespace cn // Checkpoint consensus configuration (Testnet - relaxed for smaller network) const uint32_t CKPT_MIN_CONSENSUS_PEERS_TESTNET = 1; // M: minimum agreements required (testnet) - const uint32_t CKPT_CONSENSUS_PEERS_TESTNET = 1; // K: total peers to sample (testnet) + const uint32_t CKPT_CONSENSUS_PEERS_TESTNET = 3; // K: total peers to sample (testnet) - increased to ask multiple peers const uint32_t CKPT_MIN_DIVERSE_NETWORKS_TESTNET = 1; // n: minimum distinct /16 networks required (testnet) const char P2P_STAT_TRUSTED_PUB_KEY[] = "f7061e9a5f0d30549afde49c9bfbaa52ac60afdc46304642b460a9ea34bf7a4e"; diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index d376cabfa..8ef901de9 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -268,6 +268,14 @@ bool ChunkValidationManager::validate_unverified_chunks() logger(INFO) << "Found " << eligible_peers.size() << " eligible peer(s) for chunk validation " << "(uptime > " << min_uptime_blocks << " blocks, support version 2+)"; + // Log all eligible peers for debugging + logger(DEBUGGING) << "Eligible peers list:"; + m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { + if (std::find(eligible_peers.begin(), eligible_peers.end(), peer_id) != eligible_peers.end()) { + logger(DEBUGGING) << " - Peer " << peer_id << " " << ctx << " (version " << static_cast(ctx.version) << ")"; + } + }); + // Validate chunks in chronological order (oldest first) // This ensures we catch divergences at the root cause for (uint32_t chunk_index : unverified_chunks) @@ -367,6 +375,12 @@ bool ChunkValidationManager::validate_unverified_chunks() logger(INFO) << "Sampled " << sampled_peers.size() << " peer(s) from " << distinct_networks << " distinct network(s) (requirement: " << req.min_diverse_networks << " networks)"; + // Log which peers were selected for debugging + logger(DEBUGGING) << "Sampled peers for chunk " << chunk_index << ":"; + for (uint64_t peer_id : sampled_peers) { + logger(DEBUGGING) << " - Selected peer " << peer_id; + } + // Send async requests to sampled peers uint64_t request_time = time(nullptr); uint32_t requests_sent = 0; From b1cf02103d2532ce52697b1659ff29f56cdc67e8 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 4 Mar 2026 22:02:27 -0500 Subject: [PATCH 22/56] clean logging for checkpoint chunk --- src/CryptoNoteConfig.h | 4 +- src/CryptoNoteCore/Blockchain.cpp | 64 +++++------ src/CryptoNoteCore/CheckpointsList.cpp | 5 - .../CryptoNoteProtocolHandler.cpp | 17 ++- .../CryptoNoteProtocolHandlerChunk.cpp | 100 ++++++++---------- 5 files changed, 78 insertions(+), 112 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index 3535e2fc5..9a00685fa 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -217,12 +217,12 @@ namespace cn // K = total peers to sample // n = minimum distinct /16 networks required const uint32_t CKPT_MIN_CONSENSUS_PEERS = 3; // M: minimum agreements required - const uint32_t CKPT_CONSENSUS_PEERS = 5; // K: total peers to sample + const uint32_t CKPT_CONSENSUS_PEERS = 5; // K: minimun total peers to sample const uint32_t CKPT_MIN_DIVERSE_NETWORKS = 2; // n: minimum distinct /16 networks required // Checkpoint consensus configuration (Testnet - relaxed for smaller network) const uint32_t CKPT_MIN_CONSENSUS_PEERS_TESTNET = 1; // M: minimum agreements required (testnet) - const uint32_t CKPT_CONSENSUS_PEERS_TESTNET = 3; // K: total peers to sample (testnet) - increased to ask multiple peers + const uint32_t CKPT_CONSENSUS_PEERS_TESTNET = 1; // K: minimum total peers to sample (testnet) - increased to ask multiple peers const uint32_t CKPT_MIN_DIVERSE_NETWORKS_TESTNET = 1; // n: minimum distinct /16 networks required (testnet) const char P2P_STAT_TRUSTED_PUB_KEY[] = "f7061e9a5f0d30549afde49c9bfbaa52ac60afdc46304642b460a9ea34bf7a4e"; diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 3283186b0..6ff7dd37c 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2011-2017 The Cryptonote developers // Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs // Copyright (c) 2018-2020 Karbo developers -// Copyright (c) 2018-2025 Conceal Network & Conceal Devs +// Copyright (c) 2018-2026 Conceal Network & Conceal Devs // // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -785,26 +785,21 @@ namespace cn { // Check if chunk already exists in memory (from previous session) crypto::Hash existing_chunk_hash = m_checkpoints.get_chunk_hash(chunk_idx); - if (existing_chunk_hash == NULL_HASH) + if (existing_chunk_hash != NULL_HASH) { - if (m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) - { - logger(INFO) << "Created checkpoint chunk " << chunk_idx - << " (blocks " << chunk_start_height << "-" << chunk_end_height - << ") - stored in memory, awaiting P2P validation"; - // NOTE: We do NOT call add_verified_chunk_to_file() here - // These chunks need peer consensus before being saved to checkpoint.dat - } - else - { - logger(WARNING) << "Failed to create chunk " << chunk_idx << " - blockchain mismatch detected"; - break; - } + logger(DEBUGGING) << "Chunk " << chunk_idx << " already exists in memory (from previous session), awaiting P2P validation"; + continue; // Skip to next chunk } - else + + // Create new chunk + if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) { - logger(DEBUGGING) << "Chunk " << chunk_idx << " already exists in memory (from previous session), awaiting P2P validation"; + logger(WARNING) << "Failed to create chunk " << chunk_idx << " - blockchain mismatch detected"; + break; } + // Chunk created and hash computed - logging is done in CheckpointsList.cpp + // NOTE: We do NOT call add_verified_chunk_to_file() here + // These chunks need peer consensus before being saved to checkpoint.dat } else { @@ -941,26 +936,21 @@ namespace cn { // Check if chunk already exists in memory (from previous session) crypto::Hash existing_chunk_hash = m_checkpoints.get_chunk_hash(chunk_idx); - if (existing_chunk_hash == NULL_HASH) + if (existing_chunk_hash != NULL_HASH) { - if (m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) - { - logger(INFO) << "Created checkpoint chunk " << chunk_idx - << " (blocks " << chunk_start_height << "-" << chunk_end_height - << ") - stored in memory, awaiting P2P validation"; - // NOTE: We do NOT call add_verified_chunk_to_file() here - // These chunks need peer consensus before being saved to checkpoint.dat - } - else - { - logger(WARNING) << "Failed to create chunk " << chunk_idx << " - blockchain mismatch detected"; - break; - } + logger(DEBUGGING) << "Chunk " << chunk_idx << " already exists in memory (from previous session), awaiting P2P validation"; + continue; // Skip to next chunk } - else + + // Create new chunk + if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) { - logger(DEBUGGING) << "Chunk " << chunk_idx << " already exists in memory (from previous session), awaiting P2P validation"; + logger(WARNING) << "Failed to create chunk " << chunk_idx << " - blockchain mismatch detected"; + break; } + // Chunk created and hash computed - logging is done in CheckpointsList.cpp + // NOTE: We do NOT call add_verified_chunk_to_file() here + // These chunks need peer consensus before being saved to checkpoint.dat } else { @@ -3354,13 +3344,7 @@ namespace cn return m_blockIndex.getBlockIds(startHeight, maxCount); }; - if (m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) - { - logger(INFO, BRIGHT_GREEN) << "Successfully computed chunk " << chunk_index - << " hash (blocks " << chunk_start_height << " to " << chunk_end_height - << "). Chunk stored in memory, awaiting peer consensus before saving to checkpoint.dat."; - } - else + if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) { logger(WARNING, BRIGHT_YELLOW) << "Failed to compute chunk " << chunk_index << " hash at height " << current_height; diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 3fa6a8bad..b6a0fb306 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -1306,11 +1306,6 @@ namespace cn { << ") for consensus (need K=" << req.min_peers << " for " << (m_testnet ? "testnet" : "mainnet") << ")"; } - logger(INFO) << "Consensus requirements (" << (m_testnet ? "testnet" : "mainnet") - << "): M=" << req.min_agreements - << ", K=" << req.min_peers << ", n=" << req.min_diverse_networks - << " (have " << available_peers << " available peers)"; - return req; } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 0cc731921..08ea99999 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -1346,8 +1346,6 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE return 1; } - logger(INFO) << context << " Received chunk hash response from peer " << peer_id << " for chunk " << arg.chunk_index; - // Check if we're still validating this chunk (late responses might arrive after timeout) bool still_validating = m_chunkValidationManager->is_chunk_being_validated(arg.chunk_index); @@ -1355,23 +1353,22 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE // NOTE: NULL_HASH is a valid response (means peer doesn't have this chunk) m_chunkValidationManager->store_chunk_hash_response(peer_id, arg.chunk_index, arg.chunk_hash); - // Log if this is a late response (arrived after timeout) + // Log if this is a late response (arrived after timeout) - important to know why consensus might fail if (!still_validating) { - logger(DEBUGGING) << context << " Received chunk hash response from peer " << peer_id - << " for chunk " << arg.chunk_index - << " (validation may have completed - response will be used in next validation attempt)"; + logger(WARNING) << context << " Late chunk hash response for chunk " << arg.chunk_index + << " from peer " << peer_id << " (validation may have completed)"; } if (arg.chunk_hash == NULL_HASH) { - logger(INFO) << context << " Received chunk hash response for chunk " << arg.chunk_index - << ": peer does not have this chunk (NULL_HASH)"; + logger(INFO) << context << " Received chunk " << arg.chunk_index << " hash from peer " << peer_id + << ": peer does not have this chunk (NULL_HASH)"; } else { - logger(INFO) << context << " Received chunk hash response for chunk " << arg.chunk_index - << " with hash " << arg.chunk_hash; + logger(INFO) << context << " Received chunk " << arg.chunk_index << " hash from peer " << peer_id + << " with hash " << arg.chunk_hash; } return 1; } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index 8ef901de9..ba4d79641 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -1,6 +1,6 @@ // Copyright (c) 2011-2017 The Cryptonote developers // Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -// Copyright (c) 2018-2023 Conceal Network & Conceal Devs +// Copyright (c) 2018-2026 Conceal Network & Conceal Devs // // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -89,18 +89,12 @@ bool ChunkValidationManager::send_chunk_hash_request_async(uint64_t peer_id, uin NOTIFY_REQUEST_CHUNK_HASH::request req; req.chunk_index = chunk_index; - logger(INFO) << "Sending async chunk hash request for chunk " << chunk_index << " to peer " << peer_id - << " " << *peer_context; - bool sent = post_notify(*m_p2p, req, *peer_context); if (!sent) { - logger(WARNING) << "Failed to send async chunk hash request to peer " << peer_id << " " << *peer_context - << " for chunk " << chunk_index; + logger(WARNING) << "[Chunk Validation] Failed to send chunk " << chunk_index << " hash request to peer " << peer_id; return false; } - logger(INFO) << "Successfully sent async chunk hash request for chunk " << chunk_index - << " to peer " << peer_id << " " << *peer_context; return true; } @@ -136,7 +130,7 @@ bool ChunkValidationManager::validate_unverified_chunks() // Validation can happen during sync as long as we have peers if (m_peersCount.load() == 0) { - logger(DEBUGGING) << "Cannot validate chunks: no peers connected yet"; + logger(DEBUGGING) << "[Chunk Validation] No peers available for validation"; return false; } @@ -202,15 +196,13 @@ bool ChunkValidationManager::validate_unverified_chunks() m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { total_connected++; }); - logger(INFO) << "Checking " << total_connected << " connected peer(s) for chunk validation eligibility"; + logger(INFO) << "[Chunk Validation] Checking " << total_connected << " connected peers for eligibility"; m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { // Only consider peers that support chunk-based checkpoints if (ctx.version < cn::P2P_CHECKPOINT_LIST_VERSION) { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: P2P version " - << static_cast(ctx.version) << " < " << cn::P2P_CHECKPOINT_LIST_VERSION - << " (does not support chunk-based checkpoints)"; + logger(DEBUGGING) << "[Chunk Validation] Peer " << peer_id << " filtered: P2P version " << ctx.version << " < required version"; return; // Skip old version peers } @@ -220,8 +212,7 @@ bool ChunkValidationManager::validate_unverified_chunks() ctx.m_state != CryptoNoteConnectionContext::state_idle && ctx.m_state != CryptoNoteConnectionContext::state_synchronizing) { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: state " - << get_protocol_state_string(ctx.m_state) << " (need normal, idle, or synchronizing)"; + logger(DEBUGGING) << "[Chunk Validation] Peer " << peer_id << " filtered: state " << get_protocol_state_string(ctx.m_state) << " (need normal, idle, or synchronizing)"; return; // Skip peers that aren't in a usable state } @@ -231,7 +222,7 @@ bool ChunkValidationManager::validate_unverified_chunks() time_t connection_duration = time_now - ctx.m_started; if (connection_duration < 0) { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: invalid connection time"; + logger(DEBUGGING) << "[Chunk Validation] Peer " << peer_id << " filtered: invalid connection time"; return; // Invalid connection time } @@ -240,24 +231,20 @@ bool ChunkValidationManager::validate_unverified_chunks() // Check if peer meets minimum uptime requirement if (peer_uptime_blocks < min_uptime_blocks) { - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " filtered: uptime " - << peer_uptime_blocks << " blocks < " << min_uptime_blocks << " blocks required"; + logger(DEBUGGING) << "[Chunk Validation] Peer " << peer_id << " filtered: uptime " << peer_uptime_blocks << " blocks < required"; return; // Peer doesn't meet uptime requirement } // Peer is eligible - logger(DEBUGGING) << "Peer " << peer_id << " " << ctx << " is ELIGIBLE for chunk validation: P2P version " - << static_cast(ctx.version) << ", state " - << get_protocol_state_string(ctx.m_state) << ", uptime " - << peer_uptime_blocks << " blocks"; + logger(DEBUGGING) << "[Chunk Validation] Peer " << peer_id << " is ELIGIBLE (version: " << ctx.version << ", state: " + << get_protocol_state_string(ctx.m_state) << ", uptime: " << peer_uptime_blocks << " blocks)"; eligible_peers.push_back(peer_id); peer_network_16[peer_id] = CheckpointList::get_network_16(ctx.m_remote_ip); }); if (eligible_peers.empty()) { - logger(INFO) << "No eligible peers for chunk validation (need uptime > " - << min_uptime_blocks << " blocks, version 2+, and in normal/idle/synchronizing state)"; + logger(INFO) << "[Chunk Validation] No eligible peers found (need uptime > " << min_uptime_blocks << " blocks)"; logger(INFO) << "Note: Only ACTIVE CONNECTIONS are considered, not peers in peerlist. " << "Use 'print_cn' command to see active connections. " << "To force connection to a peer, use --add-priority-node :"; @@ -265,14 +252,26 @@ bool ChunkValidationManager::validate_unverified_chunks() return false; } - logger(INFO) << "Found " << eligible_peers.size() << " eligible peer(s) for chunk validation " - << "(uptime > " << min_uptime_blocks << " blocks, support version 2+)"; + logger(INFO) << "[Chunk Validation] Found " << eligible_peers.size() << " eligible peers (uptime > " << min_uptime_blocks << " blocks)"; + + // Calculate consensus requirements (M, K, n) based on available eligible peers + CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(eligible_peers.size()); + + // STEP 1: Check if we have enough eligible peers to meet K requirement + // If not enough eligible peers, break and retry later when more peers become eligible + if (eligible_peers.size() < req.min_peers) + { + logger(INFO) << "[Chunk Validation] Not enough eligible peers: have " << eligible_peers.size() + << ", need K=" << req.min_peers << ". Will retry when more peers become available."; + return false; // Will retry via rate limiting mechanism (5 minute wait) + } + // Log all eligible peers for debugging - logger(DEBUGGING) << "Eligible peers list:"; + logger(DEBUGGING) << "[Chunk Validation] Eligible peers list:"; m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { if (std::find(eligible_peers.begin(), eligible_peers.end(), peer_id) != eligible_peers.end()) { - logger(DEBUGGING) << " - Peer " << peer_id << " " << ctx << " (version " << static_cast(ctx.version) << ")"; + logger(DEBUGGING) << " - Peer " << peer_id << " (version: " << ctx.version << ")"; } }); @@ -301,7 +300,7 @@ bool ChunkValidationManager::validate_unverified_chunks() crypto::Hash local_chunk_hash = m_core.getCheckpointList().get_chunk_hash(chunk_index); if (local_chunk_hash == NULL_HASH) { - logger(WARNING) << "Cannot validate chunk " << chunk_index << ": chunk hash not found in memory"; + logger(WARNING) << "[Chunk Validation] Cannot validate chunk " << chunk_index << ": chunk hash not found in memory"; { std::lock_guard lock(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; @@ -309,15 +308,14 @@ bool ChunkValidationManager::validate_unverified_chunks() continue; } - logger(INFO) << "Validating chunk " << chunk_index << " (oldest unverified chunk) " - << "with " << eligible_peers.size() << " eligible peer(s)"; + logger(INFO) << "[Chunk Validation] Validating chunk " << chunk_index << " with " << eligible_peers.size() << " eligible peers"; // Check if this chunk is already being validated asynchronously { std::lock_guard lock(m_pending_validations_mutex); if (m_pending_validations.find(chunk_index) != m_pending_validations.end()) { - logger(DEBUGGING) << "Chunk " << chunk_index << " is already being validated asynchronously, skipping"; + logger(DEBUGGING) << "[Chunk Validation] Chunk " << chunk_index << " is already being validated, skipping"; { std::lock_guard lock2(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; @@ -328,9 +326,6 @@ bool ChunkValidationManager::validate_unverified_chunks() // Calculate consensus requirements (M, K, n) CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(eligible_peers.size()); - logger(INFO) << "Consensus requirements (testnet): M=" << req.min_agreements - << ", K=" << req.min_peers << ", n=" << req.min_diverse_networks - << " (have " << eligible_peers.size() << " available peers)"; // Sample K peers with network diversity using shared utility function auto getPeerNetwork16 = [&peer_network_16](uint64_t peer_id) -> uint32_t { @@ -348,7 +343,7 @@ bool ChunkValidationManager::validate_unverified_chunks() if (sampled_peers.empty()) { - logger(WARNING) << "Could not sample any peers for chunk " << chunk_index << " validation"; + logger(WARNING) << "[Chunk Validation] Could not sample any peers for chunk " << chunk_index << " validation"; { std::lock_guard lock(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; @@ -361,10 +356,8 @@ bool ChunkValidationManager::validate_unverified_chunks() if (distinct_networks < req.min_diverse_networks) { - logger(WARNING) << "Could not achieve network diversity for chunk " << chunk_index - << " validation: have " << distinct_networks - << " distinct networks, need " << req.min_diverse_networks - << ". Sampled " << sampled_peers.size() << " peer(s)."; + logger(WARNING) << "[Chunk Validation] Could not achieve network diversity for chunk " << chunk_index + << ": have " << distinct_networks << " distinct networks, need " << req.min_diverse_networks; { std::lock_guard lock(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; @@ -372,11 +365,11 @@ bool ChunkValidationManager::validate_unverified_chunks() continue; } - logger(INFO) << "Sampled " << sampled_peers.size() << " peer(s) from " << distinct_networks - << " distinct network(s) (requirement: " << req.min_diverse_networks << " networks)"; + logger(INFO) << "[Chunk Validation] Sampled " << sampled_peers.size() << " peers from " << distinct_networks + << " distinct networks (requirement: " << req.min_diverse_networks << " networks)"; // Log which peers were selected for debugging - logger(DEBUGGING) << "Sampled peers for chunk " << chunk_index << ":"; + logger(DEBUGGING) << "[Chunk Validation] Sampled peers for chunk " << chunk_index << ":"; for (uint64_t peer_id : sampled_peers) { logger(DEBUGGING) << " - Selected peer " << peer_id; } @@ -394,7 +387,7 @@ bool ChunkValidationManager::validate_unverified_chunks() if (requests_sent == 0) { - logger(WARNING) << "Failed to send any async requests for chunk " << chunk_index; + logger(WARNING) << "[Chunk Validation] Failed to send any async requests for chunk " << chunk_index; { std::lock_guard lock(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; @@ -416,9 +409,7 @@ bool ChunkValidationManager::validate_unverified_chunks() m_pending_validations[chunk_index] = pending; } - logger(INFO) << "Sent async chunk hash requests for chunk " << chunk_index - << " to " << requests_sent << " peer(s). " - << "Will check for consensus after 2 minutes."; + logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << requests_sent << " peers. Checking consensus in 2 minutes."; // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) { @@ -459,8 +450,7 @@ void ChunkValidationManager::check_pending_chunk_validations() } // 2 minutes have passed - check for consensus - logger(INFO) << "Checking consensus for chunk " << chunk_index - << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; + logger(INFO) << "[Chunk Validation] Checking consensus for chunk " << chunk_index << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; // Collect responses from requested peers std::unordered_map> hash_votes; // hash -> vote count @@ -499,7 +489,7 @@ void ChunkValidationManager::check_pending_chunk_validations() // Calculate consensus requirements CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(pending.requested_peers.size()); - logger(INFO) << "Chunk " << chunk_index << " consensus check: received " << responses_received + logger(INFO) << "[Chunk Validation] Chunk " << chunk_index << " consensus check: received " << responses_received << " response(s) from " << pending.requested_peers.size() << " requested peer(s). " << "Agreements: " << agreements << " (need M=" << req.min_agreements << "), " << "NULL_HASH responses: " << null_hash_responses; @@ -508,14 +498,14 @@ void ChunkValidationManager::check_pending_chunk_validations() if (agreements >= req.min_agreements) { // Consensus reached - save to checkpoint.dat - logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index - << " validated via peer consensus (" << agreements - << " agreements, need M=" << req.min_agreements << ")"; + logger(INFO, BRIGHT_GREEN) << "[Chunk Validation] Chunk " << chunk_index + << " validated via peer consensus (" << agreements + << " agreements, need M=" << req.min_agreements << ")"; if (m_core.getCheckpointList().add_verified_chunk_to_file(chunk_index)) { - logger(INFO, BRIGHT_GREEN) << "Chunk " << chunk_index - << " saved to checkpoint.dat"; + logger(INFO, BRIGHT_GREEN) << "[Chunk Validation] Chunk " << chunk_index + << " saved to checkpoint.dat"; // Remove pending validation it = m_pending_validations.erase(it); From 2e118fac01f5611b7b84328e8f79973d4cf970a6 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 4 Mar 2026 22:03:23 -0500 Subject: [PATCH 23/56] clarify logging for IGD port mapper --- src/P2p/NetNode.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/P2p/NetNode.cpp b/src/P2p/NetNode.cpp index 35ec0eb3b..4651f7963 100644 --- a/src/P2p/NetNode.cpp +++ b/src/P2p/NetNode.cpp @@ -64,13 +64,16 @@ size_t get_random_index_with_fixed_probability(size_t max_index) { void addPortMapping(const logging::LoggerRef& logger, uint32_t port) { // Add UPnP port mapping - logger(INFO) << "Attempting to add IGD port mapping."; - int result; - UPNPDev *deviceList = upnpDiscover(1000, nullptr, nullptr, 0, 0, 2, &result); + logger(INFO) << "Attempting to add IGD port mapping for port " << port << "."; + int discoverErr = 0; + UPNPDev *deviceList = upnpDiscover(1000, nullptr, nullptr, 0, 0, 2, &discoverErr); + logger(DEBUGGING) << "upnpDiscover returned devlist=" << (void*)deviceList + << " error=" << discoverErr; UPNPUrls urls; IGDdatas igdData; char lanAddress[64]; - result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); + int result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); + logger(DEBUGGING) << "UPNP_GetValidIGD result=" << result; freeUPNPDevlist(deviceList); if (result != 0) { if (result == 1) { @@ -80,19 +83,22 @@ void addPortMapping(const logging::LoggerRef& logger, uint32_t port) { portString.str().c_str(), lanAddress, "conceal", "TCP", nullptr, "0") != 0) { logger(ERROR) << "UPNP port mapping failed."; } else { - logger(INFO, BRIGHT_GREEN) << "Added IGD port mapping."; + logger(INFO, BRIGHT_GREEN) << "Added IGD port mapping for port " << port << "."; } } else if (result == 2) { logger(INFO) << "IGD was found but reported as not connected."; } else if (result == 3) { logger(INFO) << "UPnP device was found but not recognized as IGD."; } else { - logger(ERROR) << "UPNP_GetValidIGD returned an unknown result code."; + logger(ERROR) << "UPNP_GetValidIGD returned an unknown result code: " << result; } FreeUPNPUrls(&urls); } else { - logger(INFO) << "No IGD was found."; + logger(INFO) << "No IGD was found." + << "If incoming connections fail, verify router port " + << port + << " and firewall rules."; } } From 8b9fcdb26a1f46ff81b340cce6e46790988f3d40 Mon Sep 17 00:00:00 2001 From: acktarius Date: Thu, 5 Mar 2026 10:10:45 -0500 Subject: [PATCH 24/56] fix potential memory leak --- src/Platform/Linux/System/Dispatcher.cpp | 6 ++++++ src/Platform/OSX/System/Dispatcher.cpp | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/Platform/Linux/System/Dispatcher.cpp b/src/Platform/Linux/System/Dispatcher.cpp index 76241e0bd..07b6c4e5a 100644 --- a/src/Platform/Linux/System/Dispatcher.cpp +++ b/src/Platform/Linux/System/Dispatcher.cpp @@ -133,6 +133,12 @@ Dispatcher::~Dispatcher() { delete ucontext; } + // Clean up mainContext.ucontext allocated in constructor + if (mainContext.ucontext != nullptr) { + delete static_cast(mainContext.ucontext); + mainContext.ucontext = nullptr; + } + while (!timers.empty()) { int result = ::close(timers.top()); assert(result == 0); diff --git a/src/Platform/OSX/System/Dispatcher.cpp b/src/Platform/OSX/System/Dispatcher.cpp index c3a1e6197..5d82eaa99 100644 --- a/src/Platform/OSX/System/Dispatcher.cpp +++ b/src/Platform/OSX/System/Dispatcher.cpp @@ -124,6 +124,12 @@ Dispatcher::~Dispatcher() { delete ucontext; } + // Clean up mainContext.uctx allocated in constructor + if (mainContext.uctx != nullptr) { + delete static_cast(mainContext.uctx); + mainContext.uctx = nullptr; + } + auto result = close(kqueue); assert(result != -1); result = pthread_mutex_destroy(reinterpret_cast(this->mutex)); From b5ebdf2191626ab56ee980b31cc5489cf97422e1 Mon Sep 17 00:00:00 2001 From: acktarius Date: Thu, 5 Mar 2026 10:55:20 -0500 Subject: [PATCH 25/56] update checkpoint consensus requirements --- src/CryptoNoteConfig.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index 9a00685fa..e2ff1fa56 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -216,14 +216,14 @@ namespace cn // M = minimum agreements required (all M peers must agree) // K = total peers to sample // n = minimum distinct /16 networks required - const uint32_t CKPT_MIN_CONSENSUS_PEERS = 3; // M: minimum agreements required - const uint32_t CKPT_CONSENSUS_PEERS = 5; // K: minimun total peers to sample - const uint32_t CKPT_MIN_DIVERSE_NETWORKS = 2; // n: minimum distinct /16 networks required + const uint32_t CKPT_MIN_CONSENSUS_PEERS = 6; // M: minimum agreements required + const uint32_t CKPT_CONSENSUS_PEERS = 6; // K: minimun total peers to sample + const uint32_t CKPT_MIN_DIVERSE_NETWORKS = 3; // n: minimum distinct /16 networks required // Checkpoint consensus configuration (Testnet - relaxed for smaller network) - const uint32_t CKPT_MIN_CONSENSUS_PEERS_TESTNET = 1; // M: minimum agreements required (testnet) - const uint32_t CKPT_CONSENSUS_PEERS_TESTNET = 1; // K: minimum total peers to sample (testnet) - increased to ask multiple peers - const uint32_t CKPT_MIN_DIVERSE_NETWORKS_TESTNET = 1; // n: minimum distinct /16 networks required (testnet) + const uint32_t CKPT_MIN_CONSENSUS_PEERS_TESTNET = 2; // M: minimum agreements required (testnet) + const uint32_t CKPT_CONSENSUS_PEERS_TESTNET = 2; // K: minimum total peers to sample (testnet) - increased to ask multiple peers + const uint32_t CKPT_MIN_DIVERSE_NETWORKS_TESTNET = 2; // n: minimum distinct /16 networks required (testnet) const char P2P_STAT_TRUSTED_PUB_KEY[] = "f7061e9a5f0d30549afde49c9bfbaa52ac60afdc46304642b460a9ea34bf7a4e"; From 1f736f98ea1d20677b760c25bc8ad22a3cc9d53f Mon Sep 17 00:00:00 2001 From: acktarius Date: Sat, 7 Mar 2026 16:13:11 -0500 Subject: [PATCH 26/56] skipcheckpoint conversion after actual height --- src/CryptoNoteCore/Blockchain.cpp | 3 ++- src/CryptoNoteCore/CheckpointList.h | 4 +++- src/CryptoNoteCore/CheckpointsList.cpp | 21 ++++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 6ff7dd37c..6fde88690 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -621,11 +621,12 @@ namespace cn // Convert old checkpoints (individual block hashes) to new format (list hashes) // This is only needed if we're going to use old-style full checkpoint lists + // Only convert checkpoints up to current blockchain height to avoid warnings for unsynced blocks auto getBlockIdsFunc = [this](uint32_t startHeight, uint32_t maxCount) -> std::vector { return m_blockIndex.getBlockIds(startHeight, maxCount); }; - m_checkpoints.convert_old_checkpoints_to_list_hashes(getBlockIdsFunc); + m_checkpoints.convert_old_checkpoints_to_list_hashes(getBlockIdsFunc, currentHeight); } else if (currentCoveredHeight >= greatestTargetHeight && greatestTargetHeight > 0) { diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index aec291327..6145d8d65 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -33,8 +33,10 @@ namespace cn // Convert old-style checkpoints (individual block hashes) to new-style (list hashes) // This allows the same checkpoint data in CryptoNoteConfig.h to work with both systems // getBlockIdsFunc: function that returns block IDs from genesis (0) to the specified height + // max_height: Only convert checkpoints up to this height (default: UINT32_MAX = convert all) void convert_old_checkpoints_to_list_hashes( - std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc); + std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc, + uint32_t max_height = UINT32_MAX); // Chunked checkpoint methods // Generate checkpoint chunks from block IDs (used during initialization) diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index b6a0fb306..604b2d985 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -242,7 +242,8 @@ namespace cn { } void CheckpointList::convert_old_checkpoints_to_list_hashes( - std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc) + std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc, + uint32_t max_height) { // Store old targets temporarily (they're individual block hashes from CryptoNoteConfig.h) // Note: m_targets stores size (height+1) as key, so we need to convert back to get heights @@ -256,8 +257,14 @@ namespace cn { << "to new-style (list hashes) for P2P compatibility"; logger(INFO) << "Preserving " << m_old_checkpoint_hashes.size() << " old checkpoint block hashes for chunk validation"; + if (max_height != UINT32_MAX) + { + logger(DEBUGGING) << "Only converting checkpoints up to height " << max_height + << " (current blockchain height)"; + } uint32_t converted_count = 0; + uint32_t skipped_count = 0; // For each old checkpoint, compute the list hash from genesis to that height for (const auto& old_target : old_targets) @@ -265,6 +272,13 @@ namespace cn { uint32_t size = old_target.first; // This is already height+1 (stored by add_checkpoint_target) uint32_t height = size - 1; + // Skip checkpoints beyond the current blockchain height + if (height > max_height) + { + skipped_count++; + continue; + } + // Get block IDs from genesis (0) to this height std::vector blockIds = getBlockIdsFunc(0, size); @@ -312,6 +326,11 @@ namespace cn { logger(INFO) << "Converted " << converted_count << " checkpoint validation targets for P2P compatibility"; + if (skipped_count > 0) + { + logger(DEBUGGING) << "Skipped " << skipped_count + << " checkpoints beyond current blockchain height (will convert as blockchain syncs)"; + } logger(INFO) << "Old checkpoint block hashes preserved for chunk validation: " << m_old_checkpoint_hashes.size() << " checkpoints"; } From b7bbbb4ceddb876ea1b3c940e36d636592a5a45b Mon Sep 17 00:00:00 2001 From: acktarius Date: Sat, 7 Mar 2026 18:28:08 -0500 Subject: [PATCH 27/56] fix timing of pending chunk validation --- .../CryptoNoteProtocolHandlerChunk.cpp | 17 ++++++++++------- .../CryptoNoteProtocolHandlerChunk.h | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index ba4d79641..cdcd3c9cc 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -113,8 +113,11 @@ void ChunkValidationManager::store_chunk_hash_response(uint64_t peer_id, uint32_ bool ChunkValidationManager::is_chunk_being_validated(uint32_t chunk_index) const { - std::lock_guard lock(m_chunk_validation_mutex); - return (m_current_validating_chunk_index == chunk_index); + // Check if chunk is in pending validations (actual validation state) + // m_current_validating_chunk_index is cleared immediately after sending requests, + // but validation is still pending for 3 minutes, so we need to check m_pending_validations + std::lock_guard lock(m_pending_validations_mutex); + return (m_pending_validations.find(chunk_index) != m_pending_validations.end()); } bool ChunkValidationManager::validate_unverified_chunks() @@ -409,7 +412,7 @@ bool ChunkValidationManager::validate_unverified_chunks() m_pending_validations[chunk_index] = pending; } - logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << requests_sent << " peers. Checking consensus in 2 minutes."; + logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << requests_sent << " peers. Checking consensus in 3 minutes."; // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) { @@ -428,7 +431,7 @@ bool ChunkValidationManager::validate_unverified_chunks() void ChunkValidationManager::check_pending_chunk_validations() { uint64_t time_now = time(nullptr); - const uint64_t CONSENSUS_WAIT_SECONDS = 120; // 2 minutes + const uint64_t CONSENSUS_WAIT_SECONDS = 180; // 3 minutes (increased from 2 to handle network latency and peer processing) const uint64_t RETRY_DELAY_SECONDS = 60; // 1 minute delay before retry std::lock_guard lock(m_pending_validations_mutex); @@ -441,7 +444,7 @@ void ChunkValidationManager::check_pending_chunk_validations() uint64_t elapsed = time_now - pending.request_timestamp; - // Check if 2 minutes have passed since requests were sent + // Check if 3 minutes have passed since requests were sent if (elapsed < CONSENSUS_WAIT_SECONDS) { // Not enough time has passed yet, skip this validation @@ -449,7 +452,7 @@ void ChunkValidationManager::check_pending_chunk_validations() continue; } - // 2 minutes have passed - check for consensus + // 3 minutes have passed - check for consensus logger(INFO) << "[Chunk Validation] Checking consensus for chunk " << chunk_index << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; // Collect responses from requested peers @@ -603,7 +606,7 @@ void ChunkValidationManager::check_pending_chunk_validations() logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index << " to " << requests_sent << " peer(s). " - << "Will check for consensus after 2 minutes."; + << "Will check for consensus after 3 minutes."; } else { diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h index f317c1a9b..6c3299984 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h @@ -49,7 +49,7 @@ namespace cn bool validate_unverified_chunks(); // Check pending chunk validations for consensus (called periodically) - // Checks if we have M identical hashes within the time window (2 minutes) + // Checks if we have M identical hashes within the time window (3 minutes) // If consensus reached, processes it; if timeout, schedules retry void check_pending_chunk_validations(); From 2ba5b4d7ba91ddce910ec34448d7bdd0a420a44b Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 27 Apr 2026 15:58:41 -0400 Subject: [PATCH 28/56] simplify logging * append only new chunk to checkpoint.dat --- src/CryptoNoteConfig.h | 6 +- src/CryptoNoteCore/CheckpointsList.cpp | 127 +++++++++++++----- .../CryptoNoteProtocolHandlerChunk.cpp | 3 - 3 files changed, 94 insertions(+), 42 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index e2ff1fa56..f33a2be2d 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -191,7 +191,8 @@ namespace cn const uint8_t P2P_UPGRADE_WINDOW = 2; // This defines the minimum P2P version required for lite blocks propogation - const uint8_t P2P_LITE_BLOCKS_PROPOGATION_VERSION = 2; + const uint8_t P2P_LITE_BLOCKS_PROPOGATION_VERSION = 3; + const uint8_t P2P_CHECKPOINT_LIST_VERSION = 2; const size_t P2P_LOCAL_WHITE_PEERLIST_LIMIT = 1000; @@ -239,8 +240,7 @@ namespace cn }; const std::initializer_list TESTNET_SEED_NODES = { - "161.97.145.65:15500", - "161.97.145.65:15501" + "5.189.177.60:15500" }; struct CheckpointData diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 604b2d985..77a460e6c 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -255,8 +255,6 @@ namespace cn { logger(INFO) << "Converting old-style checkpoints (individual block hashes) " << "to new-style (list hashes) for P2P compatibility"; - logger(INFO) << "Preserving " << m_old_checkpoint_hashes.size() - << " old checkpoint block hashes for chunk validation"; if (max_height != UINT32_MAX) { logger(DEBUGGING) << "Only converting checkpoints up to height " << max_height @@ -308,7 +306,7 @@ namespace cn { return; } - logger(INFO) << "Validated checkpoint at height " << height + logger(DEBUGGING) << "Validated checkpoint at height " << height << " before conversion (matches CryptoNoteConfig.h)"; } @@ -320,19 +318,19 @@ namespace cn { m_valid_point_sizes.insert(size); converted_count++; - logger(INFO) << "Converted checkpoint at height " << height + logger(DEBUGGING) << "Converted checkpoint at height " << height << " (old hash: " << old_target.second << ") to list hash: " << listHash; } logger(INFO) << "Converted " << converted_count - << " checkpoint validation targets for P2P compatibility"; + << " checkpoint validation targets for P2P compatibility" + << " (preserved " << m_old_checkpoint_hashes.size() + << " block hashes for chunk validation)"; if (skipped_count > 0) { logger(DEBUGGING) << "Skipped " << skipped_count << " checkpoints beyond current blockchain height (will convert as blockchain syncs)"; } - logger(INFO) << "Old checkpoint block hashes preserved for chunk validation: " - << m_old_checkpoint_hashes.size() << " checkpoints"; } /** @@ -602,9 +600,11 @@ namespace cn { */ bool CheckpointList::add_verified_chunk_to_file(uint32_t chunk_index) { + crypto::Hash chunk_hash; std::vector chunks_to_save; std::string file_to_save; uint32_t chunk_size; + bool append_only = false; { const std::lock_guard lock(m_chunks_lock); @@ -617,49 +617,104 @@ namespace cn { return false; } - // Get current chunks from file (only verified chunks are in file) - // We need to load existing verified chunks and append the new one - std::vector verified_chunks; - - // Load existing verified chunks from file + chunk_hash = m_chunks[chunk_index]; + file_to_save = m_save_file; + chunk_size = m_chunk_size; + + // Fast path: verified chunks are sequential, so adding the next chunk only + // needs to append one hash. Rewrites are still used for non-append cases. std::ifstream file(m_save_file, std::ios::binary | std::ios::ate); if (file.is_open()) { uint64_t fsize = file.tellg(); - file.seekg(0, std::ios::beg); - + if (fsize > 0 && fsize % sizeof(crypto::Hash) == 0) { uint32_t num_existing_chunks = static_cast(fsize / sizeof(crypto::Hash)); - verified_chunks.resize(num_existing_chunks); - file.read(reinterpret_cast(verified_chunks.data()), fsize); + append_only = (chunk_index == num_existing_chunks); + } + else if (fsize == 0) + { + append_only = (chunk_index == 0); } file.close(); } - - // Ensure verified_chunks has enough space for the new chunk - if (chunk_index >= verified_chunks.size()) + else { - verified_chunks.resize(chunk_index + 1, NULL_HASH); + append_only = (chunk_index == 0); } - // Add the verified chunk - verified_chunks[chunk_index] = m_chunks[chunk_index]; - - // Mark as confirmed (chunks in checkpoint.dat are confirmed) - m_confirmed_chunks.insert(chunk_index); - - logger(INFO) << "Adding chunk " << chunk_index - << " to checkpoint.dat"; + if (!append_only) + { + // Get current chunks from file (only verified chunks are in file) + // We need to load existing verified chunks and rewrite the file. + std::vector verified_chunks; + + std::ifstream rewrite_file(m_save_file, std::ios::binary | std::ios::ate); + if (rewrite_file.is_open()) + { + uint64_t fsize = rewrite_file.tellg(); + rewrite_file.seekg(0, std::ios::beg); + + if (fsize > 0 && fsize % sizeof(crypto::Hash) == 0) + { + uint32_t num_existing_chunks = static_cast(fsize / sizeof(crypto::Hash)); + verified_chunks.resize(num_existing_chunks); + rewrite_file.read(reinterpret_cast(verified_chunks.data()), fsize); + } + rewrite_file.close(); + } + + // Ensure verified_chunks has enough space for the new chunk + if (chunk_index >= verified_chunks.size()) + { + verified_chunks.resize(chunk_index + 1, NULL_HASH); + } + + // Add the verified chunk + verified_chunks[chunk_index] = chunk_hash; + chunks_to_save = verified_chunks; + } - // Copy data before releasing lock - chunks_to_save = verified_chunks; - file_to_save = m_save_file; - chunk_size = m_chunk_size; } // Lock released here - // Save verified chunks to file - return save_checkpoints_impl(chunks_to_save, file_to_save, chunk_size); + bool success = false; + if (append_only) + { + std::ofstream file(file_to_save, std::ios::binary | std::ios::app); + if (!file.is_open()) + { + logger(ERROR) << "Error opening checkpoint file for append: " << file_to_save; + return false; + } + + file.write(reinterpret_cast(&chunk_hash), sizeof(crypto::Hash)); + file.close(); + + if (!file) + { + logger(ERROR) << "Error appending to checkpoint file: " << file_to_save; + return false; + } + + uint32_t covered_height = (chunk_index + 1) * chunk_size; + logger(INFO) << "Appended checkpoint chunk " << chunk_index + << " to file (covers up to height " << covered_height << ")"; + success = true; + } + else + { + success = save_checkpoints_impl(chunks_to_save, file_to_save, chunk_size); + } + + if (success) + { + const std::lock_guard lock(m_chunks_lock); + // Mark as confirmed (chunks in checkpoint.dat are confirmed) + m_confirmed_chunks.insert(chunk_index); + } + + return success; } /** @@ -990,7 +1045,7 @@ namespace cn { return false; } - uint32_t covered_height = (static_cast(chunks.size()) * chunk_size) - 1; + uint32_t covered_height = static_cast(chunks.size()) * chunk_size; logger(INFO) << "Saved " << chunks.size() << " checkpoint chunks to file (covers up to height " << covered_height << ")"; return true; @@ -1710,7 +1765,7 @@ namespace cn { if (success) { - uint32_t covered_height = ((last_valid_chunk_index + 1) * chunk_size) - 1; + uint32_t covered_height = (last_valid_chunk_index + 1) * chunk_size; logger(INFO) << "Successfully truncated checkpoint.dat to chunk " << last_valid_chunk_index << " (covers up to height " << covered_height << ")"; } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index cdcd3c9cc..376ea0f83 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -507,9 +507,6 @@ void ChunkValidationManager::check_pending_chunk_validations() if (m_core.getCheckpointList().add_verified_chunk_to_file(chunk_index)) { - logger(INFO, BRIGHT_GREEN) << "[Chunk Validation] Chunk " << chunk_index - << " saved to checkpoint.dat"; - // Remove pending validation it = m_pending_validations.erase(it); From a20d901375a8806eec653ac36db066ca3f0a77b3 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 27 Apr 2026 18:04:16 -0400 Subject: [PATCH 29/56] fix * Fixed pending-validation cleanup so erased map entries are no longer referenced. * Rejected unsolicited, late, or non-requested chunk hash responses. * Counted only fresh responses for the active attempt and cleared old responses before retry. * Stored only successfully-requested peers in pending validation state. * Added vote-level network diversity checks via CheckpointList::evaluate_consensus_votes(). * Moved checkpoint file writes and rollback execution outside the pending-validation mutex. * Clarified chunk-hash serving comments for unverified in-memory chunks. * Added unit tests for stale responses, local vote diversity, valid local consensus, and divergent consensus. --- src/CryptoNoteCore/CheckpointList.h | 33 +- src/CryptoNoteCore/CheckpointsList.cpp | 114 +++- .../CryptoNoteProtocolHandler.cpp | 24 +- .../CryptoNoteProtocolHandlerChunk.cpp | 492 +++++++++--------- .../CryptoNoteProtocolHandlerChunk.h | 6 +- tests/UnitTests/Checkpoints.cpp | 110 ++++ 6 files changed, 490 insertions(+), 289 deletions(-) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index 6145d8d65..c53bb5b57 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -82,10 +82,6 @@ namespace cn // Check if node can answer checkpoint requests (requires confirmed chunks) bool can_answer_checkpoint_requests() const; - // Get the highest confirmed chunk index (for rollback safety) - // Only confirmed chunks can be used as rollback anchors - uint32_t get_highest_confirmed_chunk() const; - // Get chunks that are in memory but not yet verified (not in checkpoint.dat) // These are chunks that need peer consensus before being added to file // Returns: vector of chunk indices that need verification @@ -110,6 +106,31 @@ namespace cn uint32_t min_diverse_networks; // n: minimum distinct /16 networks required }; ConsensusRequirements calculate_consensus_requirements(size_t available_peers) const; + + struct ConsensusVote { + uint64_t peer_id; + crypto::Hash hash; + uint64_t timestamp; + }; + + struct ConsensusVoteResult { + uint32_t responses_received; + uint32_t null_hash_responses; + uint32_t agreements; + uint32_t local_diverse_networks; + crypto::Hash consensus_hash; + uint32_t consensus_hash_votes; + uint32_t consensus_hash_diverse_networks; + bool local_consensus; + bool divergent_consensus; + }; + + static ConsensusVoteResult evaluate_consensus_votes( + const std::vector& votes, + const crypto::Hash& local_hash, + const std::map& peer_network_16, + uint64_t min_response_timestamp, + const ConsensusRequirements& req); // Get minimum peer uptime requirement for checkpoint verification (network-specific) // Mainnet: 12,000 blocks (~16.7 days) @@ -161,12 +182,14 @@ namespace cn * and select any available peers to reach the target sample size. * * @param available_peers List of all available peer IDs - * @param sampled_peers Already sampled peers (will be updated) + * @param sampled_peers Already selected peers + * @param sampled_peer_ids Lookup set mirroring sampled_peers * @param target_size Target number of peers to sample */ static void fallback_peer_selection( const std::vector& available_peers, std::vector& sampled_peers, + std::unordered_set& sampled_peer_ids, size_t target_size); // Health metrics for monitoring diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 77a460e6c..e7c56219e 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -1315,24 +1316,6 @@ namespace cn { // (chunks generated from hardcoded checkpoints are automatically confirmed) return !m_confirmed_chunks.empty(); } - - uint32_t CheckpointList::get_highest_confirmed_chunk() const - { - const std::lock_guard lock(m_chunks_lock); - - if (m_confirmed_chunks.empty()) - return 0; - - // Find the highest confirmed chunk index - uint32_t highest = 0; - for (uint32_t chunk_index : m_confirmed_chunks) - { - if (chunk_index > highest) - highest = chunk_index; - } - - return highest; - } std::vector CheckpointList::get_unverified_chunks() const { @@ -1383,6 +1366,91 @@ namespace cn { return req; } + CheckpointList::ConsensusVoteResult CheckpointList::evaluate_consensus_votes( + const std::vector& votes, + const crypto::Hash& local_hash, + const std::map& peer_network_16, + uint64_t min_response_timestamp, + const ConsensusRequirements& req) + { + ConsensusVoteResult result; + result.responses_received = 0; + result.null_hash_responses = 0; + result.agreements = 0; + result.local_diverse_networks = 0; + result.consensus_hash = NULL_HASH; + result.consensus_hash_votes = 0; + result.consensus_hash_diverse_networks = 0; + result.local_consensus = false; + result.divergent_consensus = false; + + std::unordered_map> hash_votes; + std::set local_networks; + std::unordered_map, boost::hash> networks_by_hash; + + for (const ConsensusVote& vote : votes) + { + if (vote.timestamp < min_response_timestamp) + { + continue; + } + + result.responses_received++; + + if (vote.hash == NULL_HASH) + { + result.null_hash_responses++; + continue; + } + + hash_votes[vote.hash]++; + + auto network_it = peer_network_16.find(vote.peer_id); + uint32_t network = (network_it != peer_network_16.end()) ? network_it->second : 0; + + if (vote.hash == local_hash) + { + result.agreements++; + local_networks.insert(network); + } + else + { + networks_by_hash[vote.hash].insert(network); + } + } + + result.local_diverse_networks = static_cast(local_networks.size()); + result.local_consensus = (result.agreements >= req.min_agreements && + result.local_diverse_networks >= req.min_diverse_networks); + + for (const auto& vote_count : hash_votes) + { + if (vote_count.first == local_hash || vote_count.second < req.min_agreements) + { + continue; + } + + auto network_it = networks_by_hash.find(vote_count.first); + uint32_t diverse_networks = (network_it != networks_by_hash.end()) ? + static_cast(network_it->second.size()) : 0; + + if (diverse_networks < req.min_diverse_networks) + { + continue; + } + + if (!result.divergent_consensus || vote_count.second > result.consensus_hash_votes) + { + result.divergent_consensus = true; + result.consensus_hash = vote_count.first; + result.consensus_hash_votes = vote_count.second; + result.consensus_hash_diverse_networks = diverse_networks; + } + } + + return result; + } + uint32_t CheckpointList::get_network_16(uint32_t ip) { // Extract /16 network prefix: first 16 bits of IP address @@ -1408,6 +1476,7 @@ namespace cn { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution dis(0, available_peers.size() - 1); + std::unordered_set sampled_peer_ids; // Sample K peers with network diversity (max 1 vote per /16 network) while (result.sampled_peers.size() < actual_sample_size) @@ -1422,7 +1491,7 @@ namespace cn { uint64_t peer_id = available_peers[idx]; // Check if we already sampled this peer - if (std::find(result.sampled_peers.begin(), result.sampled_peers.end(), peer_id) != result.sampled_peers.end()) + if (sampled_peer_ids.find(peer_id) != sampled_peer_ids.end()) { attempts++; continue; @@ -1439,6 +1508,7 @@ namespace cn { // Accept this peer result.sampled_peers.push_back(peer_id); + sampled_peer_ids.insert(peer_id); result.network_votes[net16] = std::min(result.network_votes[net16] + 1, 1U); break; } @@ -1446,7 +1516,7 @@ namespace cn { // If we couldn't find enough diverse peers, relax diversity requirement if (result.sampled_peers.size() < actual_sample_size && attempts >= max_attempts) { - fallback_peer_selection(available_peers, result.sampled_peers, actual_sample_size); + fallback_peer_selection(available_peers, result.sampled_peers, sampled_peer_ids, actual_sample_size); } } @@ -1456,14 +1526,16 @@ namespace cn { void CheckpointList::fallback_peer_selection( const std::vector& available_peers, std::vector& sampled_peers, + std::unordered_set& sampled_peer_ids, size_t target_size) { // Fall back to any available peer (diversity requirement relaxed) for (uint64_t peer_id : available_peers) { - if (std::find(sampled_peers.begin(), sampled_peers.end(), peer_id) == sampled_peers.end()) + if (sampled_peer_ids.find(peer_id) == sampled_peer_ids.end()) { sampled_peers.push_back(peer_id); + sampled_peer_ids.insert(peer_id); if (sampled_peers.size() >= target_size) break; } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index 08ea99999..b13754480 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -1274,8 +1274,9 @@ int CryptoNoteProtocolHandler::handle_request_chunk_hash(int command, NOTIFY_REQ return 1; } - // Only answer chunk hash requests if we have confirmed chunks - // This ensures we don't serve unverified data to other nodes + // Only answer chunk hash requests after this node has at least one confirmed chunk. + // Once bootstrapped, locally computed in-memory chunks may be served so peers can + // compare them during their own consensus process. if (!m_core.getCheckpointList().can_answer_checkpoint_requests()) { logger(INFO) << context << " Cannot answer chunk hash request: no confirmed chunks available"; @@ -1287,9 +1288,8 @@ int CryptoNoteProtocolHandler::handle_request_chunk_hash(int command, NOTIFY_REQ return 1; } - // Get the chunk hash for the requested chunk index - // NOTE: We can return chunks that exist in memory (even if unverified) for P2P validation - // The requester will validate against their own chunk hash + // Return chunks that exist in memory, including not-yet-confirmed chunks. + // The requester treats this as one peer vote and validates it against consensus. crypto::Hash chunk_hash = m_core.getCheckpointList().get_chunk_hash(arg.chunk_index); NOTIFY_RESPONSE_CHUNK_HASH::request rsp; @@ -1346,18 +1346,14 @@ int CryptoNoteProtocolHandler::handle_response_chunk_hash(int command, NOTIFY_RE return 1; } - // Check if we're still validating this chunk (late responses might arrive after timeout) - bool still_validating = m_chunkValidationManager->is_chunk_being_validated(arg.chunk_index); - // Store the response in pending chunk hashes map for the consensus mechanism // NOTE: NULL_HASH is a valid response (means peer doesn't have this chunk) - m_chunkValidationManager->store_chunk_hash_response(peer_id, arg.chunk_index, arg.chunk_hash); - - // Log if this is a late response (arrived after timeout) - important to know why consensus might fail - if (!still_validating) + if (!m_chunkValidationManager->store_chunk_hash_response(peer_id, arg.chunk_index, arg.chunk_hash)) { - logger(WARNING) << context << " Late chunk hash response for chunk " << arg.chunk_index - << " from peer " << peer_id << " (validation may have completed)"; + logger(WARNING) << context << " Ignored chunk hash response for chunk " << arg.chunk_index + << " from peer " << peer_id + << " (not pending, late, or peer was not requested)"; + return 1; } if (arg.chunk_hash == NULL_HASH) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index 376ea0f83..df595932c 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -98,10 +98,23 @@ bool ChunkValidationManager::send_chunk_hash_request_async(uint64_t peer_id, uin return true; } -void ChunkValidationManager::store_chunk_hash_response(uint64_t peer_id, uint32_t chunk_index, const crypto::Hash& hash) +bool ChunkValidationManager::store_chunk_hash_response(uint64_t peer_id, uint32_t chunk_index, const crypto::Hash& hash) { uint64_t response_time = time(nullptr); { + std::lock_guard validation_lock(m_pending_validations_mutex); + auto pending_it = m_pending_validations.find(chunk_index); + if (pending_it == m_pending_validations.end()) + { + return false; + } + + const std::vector& requested_peers = pending_it->second.requested_peers; + if (std::find(requested_peers.begin(), requested_peers.end(), peer_id) == requested_peers.end()) + { + return false; + } + std::lock_guard lock(m_pending_chunk_hashes_mutex); auto key = std::make_pair(peer_id, chunk_index); ChunkHashResponse response; @@ -109,6 +122,17 @@ void ChunkValidationManager::store_chunk_hash_response(uint64_t peer_id, uint32_ response.timestamp = response_time; m_pending_chunk_hashes[key] = response; } + + return true; +} + +void ChunkValidationManager::cleanup_chunk_responses(uint32_t chunk_index, const std::vector& peer_ids) +{ + std::lock_guard lock(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : peer_ids) + { + m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + } } bool ChunkValidationManager::is_chunk_being_validated(uint32_t chunk_index) const @@ -379,18 +403,23 @@ bool ChunkValidationManager::validate_unverified_chunks() // Send async requests to sampled peers uint64_t request_time = time(nullptr); - uint32_t requests_sent = 0; + std::vector sent_peers; + std::map sent_peer_networks; for (uint64_t peer_id : sampled_peers) { if (send_chunk_hash_request_async(peer_id, chunk_index)) { - requests_sent++; + sent_peers.push_back(peer_id); + sent_peer_networks[peer_id] = getPeerNetwork16(peer_id); } } - if (requests_sent == 0) + if (sent_peers.size() < req.min_peers) { - logger(WARNING) << "[Chunk Validation] Failed to send any async requests for chunk " << chunk_index; + logger(WARNING) << "[Chunk Validation] Only sent " << sent_peers.size() + << " async request(s) for chunk " << chunk_index + << ", need K=" << req.min_peers << " for consensus"; + cleanup_chunk_responses(chunk_index, sent_peers); { std::lock_guard lock(m_chunk_validation_mutex); m_current_validating_chunk_index = UINT32_MAX; @@ -406,13 +435,14 @@ bool ChunkValidationManager::validate_unverified_chunks() pending.request_timestamp = request_time; pending.attempt_start_time = request_time; pending.attempt_number = 1; - pending.requested_peers = sampled_peers; + pending.requested_peers = sent_peers; + pending.peer_network_16 = sent_peer_networks; pending.local_hash = local_chunk_hash; pending.is_first_attempt = true; m_pending_validations[chunk_index] = pending; } - logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << requests_sent << " peers. Checking consensus in 3 minutes."; + logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << sent_peers.size() << " peers. Checking consensus in 3 minutes."; // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) { @@ -433,304 +463,270 @@ void ChunkValidationManager::check_pending_chunk_validations() uint64_t time_now = time(nullptr); const uint64_t CONSENSUS_WAIT_SECONDS = 180; // 3 minutes (increased from 2 to handle network latency and peer processing) const uint64_t RETRY_DELAY_SECONDS = 60; // 1 minute delay before retry - - std::lock_guard lock(m_pending_validations_mutex); - - // Iterate through pending validations - for (auto it = m_pending_validations.begin(); it != m_pending_validations.end();) + + struct PendingAction { - uint32_t chunk_index = it->first; - PendingChunkValidation& pending = it->second; - - uint64_t elapsed = time_now - pending.request_timestamp; - - // Check if 3 minutes have passed since requests were sent - if (elapsed < CONSENSUS_WAIT_SECONDS) + enum Type { - // Not enough time has passed yet, skip this validation - ++it; - continue; - } - - // 3 minutes have passed - check for consensus - logger(INFO) << "[Chunk Validation] Checking consensus for chunk " << chunk_index << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; - - // Collect responses from requested peers - std::unordered_map> hash_votes; // hash -> vote count - uint32_t agreements = 0; - uint32_t null_hash_responses = 0; - uint32_t responses_received = 0; - + ADD_VERIFIED_CHUNK, + ROLLBACK_DIVERGENT_CHUNK + }; + + Type type; + uint32_t chunk_index; + uint32_t rollback_height; + uint32_t last_valid_chunk; + }; + + std::vector actions; + + { + std::lock_guard lock(m_pending_validations_mutex); + + // Iterate through pending validations + for (auto it = m_pending_validations.begin(); it != m_pending_validations.end();) { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); + uint32_t chunk_index = it->first; + PendingChunkValidation& pending = it->second; - for (uint64_t peer_id : pending.requested_peers) - { - auto response_it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); - if (response_it != m_pending_chunk_hashes.end()) - { - crypto::Hash peer_hash = response_it->second.hash; - responses_received++; - - if (peer_hash == NULL_HASH) - { - null_hash_responses++; - continue; - } - - // Count votes for this hash - hash_votes[peer_hash]++; - - if (peer_hash == pending.local_hash) - { - agreements++; - } - } - } - } - - // Calculate consensus requirements - CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(pending.requested_peers.size()); - - logger(INFO) << "[Chunk Validation] Chunk " << chunk_index << " consensus check: received " << responses_received - << " response(s) from " << pending.requested_peers.size() << " requested peer(s). " - << "Agreements: " << agreements << " (need M=" << req.min_agreements << "), " - << "NULL_HASH responses: " << null_hash_responses; - - // Check if we have M agreements (consensus reached) - if (agreements >= req.min_agreements) - { - // Consensus reached - save to checkpoint.dat - logger(INFO, BRIGHT_GREEN) << "[Chunk Validation] Chunk " << chunk_index - << " validated via peer consensus (" << agreements - << " agreements, need M=" << req.min_agreements << ")"; + uint64_t elapsed = time_now - pending.request_timestamp; - if (m_core.getCheckpointList().add_verified_chunk_to_file(chunk_index)) + // Check if 3 minutes have passed since requests were sent + if (elapsed < CONSENSUS_WAIT_SECONDS) { - // Remove pending validation - it = m_pending_validations.erase(it); - - // Clean up responses for this chunk - { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - for (uint64_t peer_id : pending.requested_peers) - { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } - } - continue; - } - else - { - logger(ERROR) << "Failed to save validated chunk " << chunk_index << " to checkpoint.dat"; - // Remove pending validation anyway (we'll retry later) - it = m_pending_validations.erase(it); + // Not enough time has passed yet, skip this validation + ++it; continue; } - } - - // Check if we have M peers agreeing on a DIFFERENT hash (actual divergence) - crypto::Hash consensus_hash = NULL_HASH; - uint32_t max_votes = 0; - for (const auto& vote : hash_votes) - { - if (vote.first != pending.local_hash && vote.second > max_votes) - { - max_votes = vote.second; - consensus_hash = vote.first; - } - } - - bool has_divergence = (consensus_hash != NULL_HASH && max_votes >= req.min_agreements); - - // If all responses are NULL_HASH or missing, peers don't have this chunk yet (not a divergence) - if (responses_received == 0 || (responses_received == null_hash_responses && !has_divergence)) - { - logger(INFO) << "Chunk " << chunk_index - << " validation: No peers have this chunk in memory yet (all returned NULL_HASH or no response). " - << "This is normal if: (1) peers are using version 1 (don't support chunk checkpoints), " - << "or (2) peers haven't created this chunk yet. " - << "Will retry validation once peers create this chunk."; - // Remove pending validation - we'll retry later when peers have the chunk - it = m_pending_validations.erase(it); + // 3 minutes have passed - check for consensus + logger(INFO) << "[Chunk Validation] Checking consensus for chunk " << chunk_index << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; - // Clean up responses + // Collect fresh responses from requested peers + std::vector votes; { std::lock_guard lock2(m_pending_chunk_hashes_mutex); + for (uint64_t peer_id : pending.requested_peers) { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); + auto response_it = m_pending_chunk_hashes.find(std::make_pair(peer_id, chunk_index)); + if (response_it != m_pending_chunk_hashes.end()) + { + CheckpointList::ConsensusVote vote; + vote.peer_id = peer_id; + vote.hash = response_it->second.hash; + vote.timestamp = response_it->second.timestamp; + votes.push_back(vote); + } } } - continue; - } - - // Consensus not reached - check if we should retry - if (pending.is_first_attempt) - { - // First attempt failed - wait 1 more minute (3 minutes total) before retry - uint64_t total_elapsed = time_now - pending.attempt_start_time; - if (total_elapsed < (CONSENSUS_WAIT_SECONDS + RETRY_DELAY_SECONDS)) - { - // Still waiting for retry delay - ++it; - continue; - } - // Retry delay passed - start second attempt - logger(INFO) << "Chunk " << chunk_index - << " consensus failed on first attempt. Starting second attempt..."; + // Calculate consensus requirements + CheckpointList::ConsensusRequirements req = m_core.getCheckpointList().calculate_consensus_requirements(pending.requested_peers.size()); + CheckpointList::ConsensusVoteResult vote_result = CheckpointList::evaluate_consensus_votes( + votes, + pending.local_hash, + pending.peer_network_16, + pending.request_timestamp, + req); - // Send new requests to same peers (or get new eligible peers) - // For now, reuse same peers - uint64_t retry_time = time_now; - uint32_t requests_sent = 0; - for (uint64_t peer_id : pending.requested_peers) - { - if (send_chunk_hash_request_async(peer_id, chunk_index)) - { - requests_sent++; - } - } + logger(INFO) << "[Chunk Validation] Chunk " << chunk_index << " consensus check: received " << vote_result.responses_received + << " fresh response(s) from " << pending.requested_peers.size() << " requested peer(s). " + << "Agreements: " << vote_result.agreements << " (need M=" << req.min_agreements + << " from n=" << req.min_diverse_networks << " networks, got n=" + << vote_result.local_diverse_networks << "), " + << "NULL_HASH responses: " << vote_result.null_hash_responses; - if (requests_sent > 0) + // Check if we have M agreements from enough networks (consensus reached) + if (vote_result.local_consensus) { - // Update pending validation for second attempt - pending.request_timestamp = retry_time; - pending.attempt_number = 2; - pending.is_first_attempt = false; - - logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index - << " to " << requests_sent << " peer(s). " - << "Will check for consensus after 3 minutes."; - } - else - { - logger(WARNING) << "Failed to send second attempt requests for chunk " << chunk_index; - // Remove pending validation + logger(INFO, BRIGHT_GREEN) << "[Chunk Validation] Chunk " << chunk_index + << " validated via peer consensus (" << vote_result.agreements + << " agreements from " << vote_result.local_diverse_networks + << " networks, need M=" << req.min_agreements << ")"; + + std::vector peers_to_cleanup = pending.requested_peers; it = m_pending_validations.erase(it); + cleanup_chunk_responses(chunk_index, peers_to_cleanup); + + PendingAction action; + action.type = PendingAction::ADD_VERIFIED_CHUNK; + action.chunk_index = chunk_index; + action.rollback_height = 0; + action.last_valid_chunk = 0; + actions.push_back(action); continue; } - } - else - { - // Second attempt also failed - check if it's actual divergence or just no responses - - // Check if we have M peers agreeing on a DIFFERENT hash (actual divergence) - crypto::Hash consensus_hash_second = NULL_HASH; - uint32_t max_votes_second = 0; - for (const auto& vote : hash_votes) - { - if (vote.first != pending.local_hash && vote.second > max_votes_second) - { - max_votes_second = vote.second; - consensus_hash_second = vote.first; - } - } - - bool has_divergence_second = (consensus_hash_second != NULL_HASH && max_votes_second >= req.min_agreements); // If all responses are NULL_HASH or missing, peers don't have this chunk yet (not a divergence) - if (responses_received == 0 || (responses_received == null_hash_responses && !has_divergence_second)) + if (vote_result.responses_received == 0 || + (vote_result.responses_received == vote_result.null_hash_responses && !vote_result.divergent_consensus)) { logger(INFO) << "Chunk " << chunk_index - << " validation: No peers have this chunk in memory yet after 2 attempts " - << "(all returned NULL_HASH or no response). " + << " validation: No peers have this chunk in memory yet (all returned NULL_HASH or no response). " << "This is normal if: (1) peers are using version 1 (don't support chunk checkpoints), " << "or (2) peers haven't created this chunk yet. " << "Will retry validation once peers create this chunk."; - // Remove pending validation - we'll retry later when peers have the chunk + std::vector peers_to_cleanup = pending.requested_peers; it = m_pending_validations.erase(it); - - // Clean up responses - { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - for (uint64_t peer_id : pending.requested_peers) - { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } - } + cleanup_chunk_responses(chunk_index, peers_to_cleanup); continue; } - // Actual divergence: M peers agree on different hash - if (has_divergence_second) + // Consensus not reached - check if we should retry + if (pending.is_first_attempt) { - logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index - << " validation FAILED: peer consensus did not agree with local chunk hash " - << "after 2 attempts. M peers (" << max_votes_second - << ") agree on different hash: " << consensus_hash_second - << " (local: " << pending.local_hash << "). " - << "This indicates blockchain divergence."; + // First attempt failed - wait 1 more minute before retry + uint64_t total_elapsed = time_now - pending.attempt_start_time; + if (total_elapsed < (CONSENSUS_WAIT_SECONDS + RETRY_DELAY_SECONDS)) + { + // Still waiting for retry delay + ++it; + continue; + } - // Calculate rollback height (bottom of the chunk) - uint32_t chunk_size = m_core.getCheckpointList().get_chunk_size(); - uint32_t rollback_height = chunk_index * chunk_size; + // Retry delay passed - start second attempt + logger(INFO) << "Chunk " << chunk_index + << " consensus failed on first attempt. Starting second attempt..."; - logger(ERROR, BRIGHT_RED) << "Rolling back blockchain to height " << rollback_height - << " (chunk " << chunk_index << " boundary) due to peer consensus divergence"; + std::vector previous_peers = pending.requested_peers; + std::map previous_peer_networks = pending.peer_network_16; + cleanup_chunk_responses(chunk_index, previous_peers); + + uint64_t retry_time = time_now; + std::vector sent_peers; + std::map sent_peer_networks; + for (uint64_t peer_id : previous_peers) + { + if (send_chunk_hash_request_async(peer_id, chunk_index)) + { + sent_peers.push_back(peer_id); + auto network_it = previous_peer_networks.find(peer_id); + sent_peer_networks[peer_id] = (network_it != previous_peer_networks.end()) ? network_it->second : 0; + } + } - // Truncate checkpoint.dat to the previous chunk (chunk_index - 1) - uint32_t last_valid_chunk = (chunk_index > 0) ? (chunk_index - 1) : 0; - if (!m_core.getCheckpointList().truncate_checkpoint_file(last_valid_chunk)) + if (sent_peers.size() >= req.min_peers) { - logger(ERROR, BRIGHT_RED) << "Failed to truncate checkpoint.dat to chunk " << last_valid_chunk; + // Update pending validation for second attempt + pending.request_timestamp = retry_time; + pending.attempt_number = 2; + pending.requested_peers = sent_peers; + pending.peer_network_16 = sent_peer_networks; + pending.is_first_attempt = false; + + logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index + << " to " << sent_peers.size() << " peer(s). " + << "Will check for consensus after 3 minutes."; } else { - logger(INFO) << "Truncated checkpoint.dat to chunk " << last_valid_chunk; + logger(WARNING) << "Failed to send enough second attempt requests for chunk " << chunk_index + << ": sent " << sent_peers.size() << ", need K=" << req.min_peers; + it = m_pending_validations.erase(it); + cleanup_chunk_responses(chunk_index, sent_peers); + continue; } - - // Rollback blockchain to the chunk boundary - if (!m_core.rollback_chain_to(rollback_height)) + } + else + { + // Second attempt also failed - check if it's actual divergence or just no responses + if (vote_result.responses_received == 0 || + (vote_result.responses_received == vote_result.null_hash_responses && !vote_result.divergent_consensus)) { - logger(ERROR, BRIGHT_RED) << "Failed to rollback blockchain to height " << rollback_height - << " - node may be in inconsistent state"; + logger(INFO) << "Chunk " << chunk_index + << " validation: No peers have this chunk in memory yet after 2 attempts " + << "(all returned NULL_HASH or no response). " + << "This is normal if: (1) peers are using version 1 (don't support chunk checkpoints), " + << "or (2) peers haven't created this chunk yet. " + << "Will retry validation once peers create this chunk."; + + std::vector peers_to_cleanup = pending.requested_peers; + it = m_pending_validations.erase(it); + cleanup_chunk_responses(chunk_index, peers_to_cleanup); + continue; } - else + + // Actual divergence: M peers agree on a different hash from enough networks + if (vote_result.divergent_consensus) { - logger(INFO, BRIGHT_GREEN) << "Successfully rolled back blockchain to height " << rollback_height; + logger(ERROR, BRIGHT_RED) << "Chunk " << chunk_index + << " validation FAILED: peer consensus did not agree with local chunk hash " + << "after 2 attempts. M peers (" << vote_result.consensus_hash_votes + << ") from " << vote_result.consensus_hash_diverse_networks + << " networks agree on different hash: " << vote_result.consensus_hash + << " (local: " << pending.local_hash << "). " + << "This indicates blockchain divergence."; + + // Calculate rollback height (bottom of the chunk) + uint32_t chunk_size = m_core.getCheckpointList().get_chunk_size(); + uint32_t rollback_height = chunk_index * chunk_size; + + logger(ERROR, BRIGHT_RED) << "Rolling back blockchain to height " << rollback_height + << " (chunk " << chunk_index << " boundary) due to peer consensus divergence"; + + PendingAction action; + action.type = PendingAction::ROLLBACK_DIVERGENT_CHUNK; + action.chunk_index = chunk_index; + action.rollback_height = rollback_height; + action.last_valid_chunk = (chunk_index > 0) ? (chunk_index - 1) : 0; + actions.push_back(action); + + std::vector peers_to_cleanup = pending.requested_peers; + it = m_pending_validations.erase(it); + cleanup_chunk_responses(chunk_index, peers_to_cleanup); + continue; } - // Remove pending validation - it = m_pending_validations.erase(it); + // Mixed responses (some NULL_HASH, some mismatches, but no diverse M agreement on a single different hash) + logger(WARNING) << "Chunk " << chunk_index + << " validation: Inconsistent results after 2 attempts. " + << "Some peers returned NULL_HASH, some returned different hashes, " + << "but no M peers from enough networks agreed on a single different hash. " + << "This may indicate network issues or peers still syncing. " + << "Will retry validation later."; - // Clean up responses - { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - for (uint64_t peer_id : pending.requested_peers) - { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } - } + std::vector peers_to_cleanup = pending.requested_peers; + it = m_pending_validations.erase(it); + cleanup_chunk_responses(chunk_index, peers_to_cleanup); continue; } - // Mixed responses (some NULL_HASH, some mismatches, but not M agreements on different hash) - logger(WARNING) << "Chunk " << chunk_index - << " validation: Inconsistent results after 2 attempts. " - << "Some peers returned NULL_HASH, some returned different hashes, " - << "but no M peers agreed on a single different hash. " - << "This may indicate network issues or peers still syncing. " - << "Will retry validation later."; - - // Remove pending validation - we'll retry later - it = m_pending_validations.erase(it); + ++it; + } + } + + for (const PendingAction& action : actions) + { + if (action.type == PendingAction::ADD_VERIFIED_CHUNK) + { + if (!m_core.getCheckpointList().add_verified_chunk_to_file(action.chunk_index)) + { + logger(ERROR) << "Failed to save validated chunk " << action.chunk_index << " to checkpoint.dat"; + } + } + else + { + if (!m_core.getCheckpointList().truncate_checkpoint_file(action.last_valid_chunk)) + { + logger(ERROR, BRIGHT_RED) << "Failed to truncate checkpoint.dat to chunk " << action.last_valid_chunk; + } + else + { + logger(INFO) << "Truncated checkpoint.dat to chunk " << action.last_valid_chunk; + } - // Clean up responses + if (!m_core.rollback_chain_to(action.rollback_height)) { - std::lock_guard lock2(m_pending_chunk_hashes_mutex); - for (uint64_t peer_id : pending.requested_peers) - { - m_pending_chunk_hashes.erase(std::make_pair(peer_id, chunk_index)); - } + logger(ERROR, BRIGHT_RED) << "Failed to rollback blockchain to height " << action.rollback_height + << " - node may be in inconsistent state"; + } + else + { + logger(INFO, BRIGHT_GREEN) << "Successfully rolled back blockchain to height " << action.rollback_height; } - continue; } - - ++it; } } diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h index 6c3299984..668ec3008 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h @@ -58,7 +58,8 @@ namespace cn bool send_chunk_hash_request_async(uint64_t peer_id, uint32_t chunk_index); // Store chunk hash response (called from message handler) - void store_chunk_hash_response(uint64_t peer_id, uint32_t chunk_index, const crypto::Hash& hash); + // Returns false for late, unsolicited, or non-requested peer responses. + bool store_chunk_hash_response(uint64_t peer_id, uint32_t chunk_index, const crypto::Hash& hash); // Check if chunk is currently being validated bool is_chunk_being_validated(uint32_t chunk_index) const; @@ -78,9 +79,12 @@ namespace cn uint64_t attempt_start_time; // When this attempt started uint32_t attempt_number; // 1 or 2 std::vector requested_peers; // Peers we requested from + std::map peer_network_16; // peer_id -> /16 network crypto::Hash local_hash; bool is_first_attempt; }; + + void cleanup_chunk_responses(uint32_t chunk_index, const std::vector& peer_ids); ICore& m_core; IP2pEndpoint* m_p2p; diff --git a/tests/UnitTests/Checkpoints.cpp b/tests/UnitTests/Checkpoints.cpp index beaa8207e..4fcbffbea 100644 --- a/tests/UnitTests/Checkpoints.cpp +++ b/tests/UnitTests/Checkpoints.cpp @@ -14,6 +14,8 @@ #include #include #include "crypto/hash.h" +#include +#include using namespace cn; @@ -26,6 +28,114 @@ class TestTransactionValidator : public cn::ITransactionValidator { bool checkTransactionSize(size_t) override { return true; } }; +namespace { + +crypto::Hash testHash(const std::string& hex) { + crypto::Hash hash = NULL_HASH; + bool parsed = common::podFromHex(hex, hash); + EXPECT_TRUE(parsed); + return hash; +} + +CheckpointList::ConsensusRequirements testConsensusRequirements() { + CheckpointList::ConsensusRequirements req; + req.min_agreements = 2; + req.min_peers = 2; + req.min_diverse_networks = 2; + return req; +} + +CheckpointList::ConsensusVote testVote(uint64_t peer_id, const crypto::Hash& hash, uint64_t timestamp) { + CheckpointList::ConsensusVote vote; + vote.peer_id = peer_id; + vote.hash = hash; + vote.timestamp = timestamp; + return vote; +} + +} + +TEST(checkpoints_consensus_votes, ignores_stale_responses) +{ + crypto::Hash local_hash = testHash("1111111111111111111111111111111111111111111111111111111111111111"); + std::vector votes; + votes.push_back(testVote(1, local_hash, 10)); + votes.push_back(testVote(2, local_hash, 10)); + + std::map peer_networks; + peer_networks[1] = 0x0101; + peer_networks[2] = 0x0202; + + CheckpointList::ConsensusVoteResult result = CheckpointList::evaluate_consensus_votes( + votes, local_hash, peer_networks, 20, testConsensusRequirements()); + + ASSERT_EQ(0, result.responses_received); + ASSERT_EQ(0, result.agreements); + ASSERT_FALSE(result.local_consensus); +} + +TEST(checkpoints_consensus_votes, requires_diverse_networks_for_local_consensus) +{ + crypto::Hash local_hash = testHash("2222222222222222222222222222222222222222222222222222222222222222"); + std::vector votes; + votes.push_back(testVote(1, local_hash, 30)); + votes.push_back(testVote(2, local_hash, 30)); + + std::map peer_networks; + peer_networks[1] = 0x0101; + peer_networks[2] = 0x0101; + + CheckpointList::ConsensusVoteResult result = CheckpointList::evaluate_consensus_votes( + votes, local_hash, peer_networks, 20, testConsensusRequirements()); + + ASSERT_EQ(2, result.responses_received); + ASSERT_EQ(2, result.agreements); + ASSERT_EQ(1, result.local_diverse_networks); + ASSERT_FALSE(result.local_consensus); +} + +TEST(checkpoints_consensus_votes, accepts_diverse_local_consensus) +{ + crypto::Hash local_hash = testHash("3333333333333333333333333333333333333333333333333333333333333333"); + std::vector votes; + votes.push_back(testVote(1, local_hash, 30)); + votes.push_back(testVote(2, local_hash, 30)); + + std::map peer_networks; + peer_networks[1] = 0x0101; + peer_networks[2] = 0x0202; + + CheckpointList::ConsensusVoteResult result = CheckpointList::evaluate_consensus_votes( + votes, local_hash, peer_networks, 20, testConsensusRequirements()); + + ASSERT_EQ(2, result.responses_received); + ASSERT_EQ(2, result.agreements); + ASSERT_EQ(2, result.local_diverse_networks); + ASSERT_TRUE(result.local_consensus); +} + +TEST(checkpoints_consensus_votes, detects_diverse_divergent_consensus) +{ + crypto::Hash local_hash = testHash("4444444444444444444444444444444444444444444444444444444444444444"); + crypto::Hash peer_hash = testHash("5555555555555555555555555555555555555555555555555555555555555555"); + std::vector votes; + votes.push_back(testVote(1, peer_hash, 30)); + votes.push_back(testVote(2, peer_hash, 30)); + + std::map peer_networks; + peer_networks[1] = 0x0101; + peer_networks[2] = 0x0202; + + CheckpointList::ConsensusVoteResult result = CheckpointList::evaluate_consensus_votes( + votes, local_hash, peer_networks, 20, testConsensusRequirements()); + + ASSERT_FALSE(result.local_consensus); + ASSERT_TRUE(result.divergent_consensus); + ASSERT_EQ(peer_hash, result.consensus_hash); + ASSERT_EQ(2, result.consensus_hash_votes); + ASSERT_EQ(2, result.consensus_hash_diverse_networks); +} + TEST(checkpoints_is_alternative_block_allowed, handles_empty_checkpoints) { logging::LoggerGroup logger; From 4521b9e295a26004209f4bcd283b3ab3e0edfb8f Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 27 Apr 2026 18:40:37 -0400 Subject: [PATCH 30/56] store block cache * add backup file blocksCache.dat also after rebuild, so it can be available after a crash --- src/CryptoNoteCore/Blockchain.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 6fde88690..a1db6699f 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -517,6 +517,12 @@ namespace cn logger(ERROR, BRIGHT_RED) << "Failed to rebuild cache"; return false; } + + if (!storeCache()) + { + logger(ERROR, BRIGHT_RED) << "Failed to save rebuilt blockchain cache"; + return false; + } } } @@ -552,6 +558,12 @@ namespace cn logger(ERROR, BRIGHT_RED) << "Failed to rebuild cache"; return false; } + + if (!storeCache()) + { + logger(ERROR, BRIGHT_RED) << "Failed to save rebuilt blockchain cache"; + return false; + } } catch (const std::exception&) { @@ -1342,13 +1354,15 @@ namespace cn BlockCacheSerializer ser(*this, getTailId(), logger.getLogger()); const std::string &blocksCacheFileName = m_currency.blocksCacheFileName(); - std::string blockCacheBkpFileName = blocksCacheFileName + ".bkp"; + const std::string blocksCachePath = appendPath(m_config_folder, blocksCacheFileName); + const std::string blockCacheBkpPath = blocksCachePath + ".bkp"; try { - std::rename(blocksCacheFileName.c_str(), blockCacheBkpFileName.c_str()); // fail here can be ignored + std::remove(blockCacheBkpPath.c_str()); // fail here can be ignored + std::rename(blocksCachePath.c_str(), blockCacheBkpPath.c_str()); // fail here can be ignored - if (!ser.save(appendPath(m_config_folder, blocksCacheFileName))) + if (!ser.save(blocksCachePath)) { logger(ERROR, BRIGHT_RED) << "Failed to save blockchain cache"; return false; From c8e02bd2387db1decbc03771fc6dc2bed4ff2410 Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 27 Apr 2026 19:51:24 -0400 Subject: [PATCH 31/56] sync ICoreStub with ICore --- tests/UnitTests/ICoreStub.cpp | 31 +++++++++++++++++++++++++++++++ tests/UnitTests/ICoreStub.h | 5 +++++ 2 files changed, 36 insertions(+) diff --git a/tests/UnitTests/ICoreStub.cpp b/tests/UnitTests/ICoreStub.cpp index de754050c..d495e7b1b 100644 --- a/tests/UnitTests/ICoreStub.cpp +++ b/tests/UnitTests/ICoreStub.cpp @@ -10,9 +10,11 @@ #include "CryptoNoteCore/IBlock.h" #include "CryptoNoteCore/VerificationContext.h" +#include ICoreStub::ICoreStub() : m_currency(cn::CurrencyBuilder(m_logger).currency()), + m_checkpoints(m_logger), topHeight(0), globalIndicesResult(false), randomOutsResult(false), @@ -22,6 +24,7 @@ ICoreStub::ICoreStub() : ICoreStub::ICoreStub(const cn::Block& genesisBlock) : m_currency(cn::CurrencyBuilder(m_logger).currency()), + m_checkpoints(m_logger), topHeight(0), globalIndicesResult(false), randomOutsResult(false), @@ -86,6 +89,10 @@ bool ICoreStub::handle_incoming_block(const cn::Block &b, cn::block_verification return false; } +cn::CheckpointList& ICoreStub::getCheckpointList() { + return m_checkpoints; +} + void ICoreStub::set_blockchain_top(uint32_t height, const crypto::Hash& top_id) { topHeight = height; topId = top_id; @@ -207,6 +214,26 @@ crypto::Hash ICoreStub::getBlockIdByHeight(uint32_t height) { return iter->second; } +std::vector ICoreStub::getBlockIds(uint32_t start_height, uint32_t end_height) { + std::vector result; + uint32_t height = start_height; + while (height <= end_height) { + auto iter = blockHashByHeightIndex.find(height); + if (iter == blockHashByHeightIndex.end()) { + break; + } + + result.push_back(iter->second); + if (height == std::numeric_limits::max()) { + break; + } + + ++height; + } + + return result; +} + bool ICoreStub::getBlockByHash(const crypto::Hash &h, cn::Block &blk) { auto iter = blocks.find(h); if (iter == blocks.end()) { @@ -375,6 +402,10 @@ bool ICoreStub::removeMessageQueue(cn::MessageQueue& mess return true; } +bool ICoreStub::rollback_chain_to(uint32_t height) { + return true; +} + void ICoreStub::setPoolChangesResult(bool result) { poolChangesResult = result; } diff --git a/tests/UnitTests/ICoreStub.h b/tests/UnitTests/ICoreStub.h index 7854d3f17..d89c6b6f3 100644 --- a/tests/UnitTests/ICoreStub.h +++ b/tests/UnitTests/ICoreStub.h @@ -9,6 +9,7 @@ #include #include "CryptoNoteCore/CryptoNoteBasic.h" +#include "CryptoNoteCore/CheckpointList.h" #include "CryptoNoteCore/ICore.h" #include "CryptoNoteCore/ICoreObserver.h" #include "CryptoNoteCore/Currency.h" @@ -59,10 +60,12 @@ class ICoreStub: public cn::ICore { bool handle_incoming_block(const cn::Block &b, cn::block_verification_context &bvc, bool control_miner, bool relay_block) override; virtual bool handle_get_objects(cn::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cn::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp) override { return false; } virtual void on_synchronized() override {} + virtual cn::CheckpointList& getCheckpointList() override; virtual bool getOutByMSigGIndex(uint64_t amount, uint64_t gindex, cn::MultisignatureOutput& out) override { return true; } virtual size_t addChain(const std::vector& chain) override; virtual crypto::Hash getBlockIdByHeight(uint32_t height) override; + virtual std::vector getBlockIds(uint32_t start_height, uint32_t end_height) override; virtual bool getBlockByHash(const crypto::Hash &h, cn::Block &blk) override; virtual bool getBlockHeight(const crypto::Hash& blockId, uint32_t& blockHeight) override; bool getTransaction(const crypto::Hash &id, cn::Transaction &tx, bool checkTxPool = false) override; @@ -90,6 +93,7 @@ class ICoreStub: public cn::ICore { virtual bool addMessageQueue(cn::MessageQueue& messageQueuePtr) override; virtual bool removeMessageQueue(cn::MessageQueue& messageQueuePtr) override; + virtual bool rollback_chain_to(uint32_t height) override; void set_blockchain_top(uint32_t height, const crypto::Hash& top_id); @@ -105,6 +109,7 @@ class ICoreStub: public cn::ICore { private: logging::ConsoleLogger m_logger; cn::Currency m_currency; + cn::CheckpointList m_checkpoints; uint32_t topHeight; crypto::Hash topId; From f574843c21f585539a8a4617f6223565be8f6e2b Mon Sep 17 00:00:00 2001 From: acktarius Date: Mon, 27 Apr 2026 20:06:19 -0400 Subject: [PATCH 32/56] fix workflow permissions --- .github/workflows/check.yml | 20 +++++++++++--------- .github/workflows/macOS.yml | 11 +++++++---- .github/workflows/ubuntu22.yml | 7 +++++-- .github/workflows/ubuntu24.yml | 9 ++++++--- .github/workflows/windows.yml | 7 +++++-- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 94eb5f4a4..b9f1de176 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,7 +7,9 @@ on: tags-ignore: - "*" # We don't want this to run on release pull_request: - +permissions: + contents: read + jobs: build-windows: name: Windows @@ -15,7 +17,7 @@ jobs: env: BOOST_ROOT: C:/local/boost_1_83_0 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Prepare version shell: powershell @@ -103,7 +105,7 @@ jobs: run: shell: msys2 {0} steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - uses: msys2/setup-msys2@v2 with: @@ -184,7 +186,7 @@ jobs: name: Ubuntu 22.04 runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Prepare version id: setup @@ -254,7 +256,7 @@ jobs: name: Ubuntu 24.04 runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Prepare version id: setup @@ -324,7 +326,7 @@ jobs: name: Ubuntu 22.04 clang runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Prepare version id: setup @@ -392,14 +394,14 @@ jobs: build-macos: name: macOS - runs-on: macos-13 + runs-on: macos-15-intel steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Prepare version id: setup run: | - os=macos-13 + os=macos-15-intel ccx_version=${GITHUB_SHA::7} release_name=ccx-cli-"$os"-dev-"$ccx_version" echo "release_name=${release_name}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 03aab0c24..06157842f 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -4,13 +4,16 @@ on: push: tags: - "*" - + +permissions: + contents: write + jobs: build-macos: name: macOS - runs-on: macos-13 + runs-on: macos-15-intel steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Build id: build @@ -36,7 +39,7 @@ jobs: echo "ccx_version=${ccx_version}" >> $GITHUB_OUTPUT - name: Create Release - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v3 with: files: ${{ steps.build.outputs.asset_path }} name: Conceal Core CLI v${{ steps.build.outputs.ccx_version }} diff --git a/.github/workflows/ubuntu22.yml b/.github/workflows/ubuntu22.yml index e82b4d8b0..19e8561ad 100644 --- a/.github/workflows/ubuntu22.yml +++ b/.github/workflows/ubuntu22.yml @@ -5,12 +5,15 @@ on: tags: - "*" +permissions: + contents: write + jobs: build-ubuntu22: name: Ubuntu 22.04 runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Build id: build @@ -43,7 +46,7 @@ jobs: echo "ccx_version=${ccx_version}" >> $GITHUB_OUTPUT - name: Create Release - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v3 with: files: ${{ steps.build.outputs.asset_path }} name: Conceal Core CLI v${{ steps.build.outputs.ccx_version }} diff --git a/.github/workflows/ubuntu24.yml b/.github/workflows/ubuntu24.yml index 2e2e338f6..d163b056c 100644 --- a/.github/workflows/ubuntu24.yml +++ b/.github/workflows/ubuntu24.yml @@ -4,13 +4,16 @@ on: push: tags: - "*" - + +permissions: + contents: write + jobs: build-ubuntu24: name: Ubuntu 24.04 runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Build id: build @@ -43,7 +46,7 @@ jobs: echo "ccx_version=${ccx_version}" >> $GITHUB_OUTPUT - name: Create Release - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v3 with: files: ${{ steps.build.outputs.asset_path }} name: Conceal Core CLI v${{ steps.build.outputs.ccx_version }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 38b3522a7..afef10d13 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -5,6 +5,9 @@ on: tags: - "*" +permissions: + contents: write + jobs: build-windows: name: Windows @@ -12,7 +15,7 @@ jobs: env: BOOST_ROOT: C:/local/boost_1_83_0 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v5 - name: Setup msbuild uses: microsoft/setup-msbuild@v2 @@ -55,7 +58,7 @@ jobs: echo "ccx_version=${ccx_version}" >> $env:GITHUB_OUTPUT - name: Create Release - uses: softprops/action-gh-release@v2.0.4 + uses: softprops/action-gh-release@v3 with: files: ${{ steps.build.outputs.asset_path }} name: Conceal Core CLI v${{ steps.build.outputs.ccx_version }} From 450eadf1f2773f37d2fc87e64b24041f8c396690 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 28 Apr 2026 14:34:37 -0400 Subject: [PATCH 33/56] refactor(tests): implement RAII-based temporary directory isolation for chaingen Introduce a unique temporary directory for each chaingen test run using boost::filesystem. This ensures that test environments are isolated from production checkpoint files and prevents blockchain file contamination between consecutive tests. A TempDirGuard RAII object is used to automatically clean up the directory upon test completion. --- .github/workflows/check.yml | 3 ++- .gitignore | 3 ++- src/CryptoNoteCore/Blockchain.cpp | 10 ++++++++-- src/CryptoNoteCore/CheckpointList.h | 18 +++++++++++------- tests/CoreTests/Chaingen.h | 20 ++++++++++++++++++++ 5 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b9f1de176..11b5d1bb6 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -7,8 +7,9 @@ on: tags-ignore: - "*" # We don't want this to run on release pull_request: + permissions: - contents: read + contents: write jobs: build-windows: diff --git a/.gitignore b/.gitignore index f42fcf92a..efb6ba307 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ compile_commands.json *.json cmake-build* -.cursorrules \ No newline at end of file +.cursorrules +CLAUDE.md \ No newline at end of file diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index a1db6699f..a2fc07ffa 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -2085,8 +2085,14 @@ namespace cn return false; } - uint32_t checkpoint_height = m_checkpoints.get_greatest_target_height(); - return checkpoint_height < block_height; + // Use the height actually covered by confirmed chunk hashes, not the + // target height we want to reach. The target comes from CHECKPOINTS in + // CryptoNoteConfig.h and can be far in the future relative to the current + // chain tip (or synthetic test blocks), which would incorrectly reject + // every alternative block. Only blocks in the confirmed checkpoint zone + // (height <= covered_height) should be denied as alternative heads. + uint32_t covered_height = m_checkpoints.get_covered_height(); + return covered_height < block_height; } bool Blockchain::handle_alternative_block(const Block &b, const crypto::Hash &id, block_verification_context &bvc, bool sendNewAlternativeBlockMessage) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index c53bb5b57..c03d555fb 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -308,8 +308,11 @@ namespace cn bool is_in_checkpoint_zone(uint32_t height) const { - // A height is in checkpoint zone if we don't have chunks covering it yet - return get_covered_height() < height; + // A height is in the checkpoint zone when it is covered by loaded chunk + // hashes — meaning it can be trusted without full ring-sig/input + // verification. Height 0 (genesis) is always outside the zone so it + // is never used to skip validation. + return height > 0 && height <= get_covered_height(); } enum check_rt @@ -328,6 +331,12 @@ namespace cn // Individual block validation isn't possible with chunks alone, but chunk integrity ensures chain correctness check_rt check_checkpoint(uint32_t height, const crypto::Hash& hv) const { + // Genesis is validated by Blockchain initialization. Core tests replace + // genesis with synthetic blocks, so do not compare height 0 with network + // checkpoint data here. + if (height == 0) + return is_checkpointed; + // FIRST: Check if this height is a hardcoded checkpoint (from CryptoNoteConfig.h) // This works for BOTH version 1 and version 2 systems // m_old_checkpoint_hashes contains individual block hashes for each checkpoint height @@ -358,11 +367,6 @@ namespace cn // If we have chunks, use chunk-based zone checking if (!m_chunks.empty()) { - // SIMPLIFIED: Calculate which chunk this height belongs to - // Block 0 (genesis) is not in any chunk, so we need to handle it separately - if (height == 0) - return is_checkpointed; // Genesis is always valid (checked separately) in blockchain.cpp line 847 - // For heights >= 1: chunk_index = (height - 1) / chunk_size uint32_t chunk_index = (height - 1) / m_chunk_size; diff --git a/tests/CoreTests/Chaingen.h b/tests/CoreTests/Chaingen.h index df514ca86..9103434c4 100644 --- a/tests/CoreTests/Chaingen.h +++ b/tests/CoreTests/Chaingen.h @@ -7,6 +7,7 @@ #include #include +#include #include "CryptoNoteCore/CoreConfig.h" #include "Common/CommandLine.h" @@ -399,9 +400,28 @@ inline bool do_replay_events(std::vector& events, t_test_class if (!r) return false; + // Each test gets its own unique temp directory so: + // 1. No production checkpoint.dat is loaded — chunk-based checkpoints + // covering small heights would bypass PoW for synthetic test blocks. + // 2. SwappedVector blockchain files from one test don't contaminate the next. + boost::filesystem::path testDir = boost::filesystem::temp_directory_path() / + boost::filesystem::unique_path("conceal_core_test_%%%%-%%%%"); + boost::filesystem::create_directories(testDir); + + // RAII guard: remove the temp dir when this scope exits (best effort). + struct TempDirGuard { + boost::filesystem::path path; + ~TempDirGuard() { + boost::system::error_code ec; + boost::filesystem::remove_all(path, ec); + } + } tempGuard{testDir}; + logging::ConsoleLogger logger; cn::CoreConfig coreConfig; coreConfig.init(vm); + coreConfig.configFolder = testDir.string(); + cn::MinerConfig emptyMinerConfig; cn::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cn::core c(validator.currency(), &pr, logger); From 120ed08f492c820c4d0ab20bc4f214e8a83a8ad0 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 28 Apr 2026 16:58:15 -0400 Subject: [PATCH 34/56] =?UTF-8?q?fix=20for=20testing=20with=20new=20checkp?= =?UTF-8?q?oints=20structure=20*=20Call=20clear=5Ftargets=5Ffor=5Ftest()?= =?UTF-8?q?=20after=20init=5Ftargets()=20to=20isolate=20the=20test=20state?= =?UTF-8?q?=20*=20Compute=20the=20real=20cn=5Ffast=5Fhash=20of=20the=20zer?= =?UTF-8?q?o-hash=20vector=20and=20use=20it=20as=20the=20target=20(instead?= =?UTF-8?q?=20of=20"00...00")=20*=20Actually=20call=20set=5Fcheckpoint=5Fl?= =?UTF-8?q?ist()=20to=20load=20m=5Fpoints,=20mirroring=20how=20old=20check?= =?UTF-8?q?point.dat=20was=20loaded=20*=20Restore=20ASSERT=5FEQ(greatest,?= =?UTF-8?q?=20N)=20sanity=20checks=20=E2=80=94=20they=20now=20pass=20becau?= =?UTF-8?q?se=20the=20state=20is=20controlled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CryptoNoteCore/Blockchain.cpp | 25 ++++++++---- src/CryptoNoteCore/CheckpointList.h | 21 +++++++++- tests/UnitTests/Checkpoints.cpp | 60 ++++++++++++++++------------- 3 files changed, 71 insertions(+), 35 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index a2fc07ffa..61705075a 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -2085,14 +2085,25 @@ namespace cn return false; } - // Use the height actually covered by confirmed chunk hashes, not the - // target height we want to reach. The target comes from CHECKPOINTS in - // CryptoNoteConfig.h and can be far in the future relative to the current - // chain tip (or synthetic test blocks), which would incorrectly reject - // every alternative block. Only blocks in the confirmed checkpoint zone - // (height <= covered_height) should be denied as alternative heads. + // Prefer chunk-based coverage (live chain): only blocks beyond the confirmed + // checkpoint zone are allowed as alternative heads. uint32_t covered_height = m_checkpoints.get_covered_height(); - return covered_height < block_height; + if (covered_height > 0) { + return covered_height < block_height; + } + + // No chunks present (e.g. unit tests, early bootstrap): fall back to + // target-based logic. Find the highest checkpoint target whose height has + // already been reached by the current blockchain. Blocks at or below that + // height cannot be alternative heads; blocks above it can. + uint32_t checkpoint_zone_height = 0; + for (const auto& target : m_checkpoints.get_checkpoint_targets()) { + uint32_t target_height = target.first; + if (target_height <= blockchain_height && target_height > checkpoint_zone_height) { + checkpoint_zone_height = target_height; + } + } + return checkpoint_zone_height < block_height; } bool Blockchain::handle_alternative_block(const Block &b, const crypto::Hash &id, block_verification_context &bvc, bool sendNewAlternativeBlockMessage) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index c03d555fb..bae9380fa 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -252,9 +252,28 @@ namespace cn bool set_checkpoint_list(std::vector&& points); bool load_checkpoints_from_file(); - // Test helper to add checkpoint targets (for unit testing only) + // Test helpers (for unit testing only) bool add_checkpoint_target_for_test(uint32_t height, const std::string& hash_str); + // Clear all checkpoint state so tests start from a known empty slate + // after init_targets() has loaded network checkpoints from CryptoNoteConfig.h + void clear_targets_for_test() + { + m_targets.clear(); + m_valid_point_sizes.clear(); + m_old_checkpoint_hashes.clear(); + m_dns_checkpoint_hashes.clear(); + { + const std::lock_guard lock(m_points_lock); + m_points.clear(); + } + { + const std::lock_guard lock(m_chunks_lock); + m_chunks.clear(); + m_confirmed_chunks.clear(); + } + } + // Get the number of chunks (not individual blocks) uint32_t get_chunk_count() const { diff --git a/tests/UnitTests/Checkpoints.cpp b/tests/UnitTests/Checkpoints.cpp index 4fcbffbea..c2ff5d427 100644 --- a/tests/UnitTests/Checkpoints.cpp +++ b/tests/UnitTests/Checkpoints.cpp @@ -165,25 +165,21 @@ TEST(checkpoints_is_alternative_block_allowed, handles_one_checkpoint) Blockchain blockchain(currency, tx_pool, logger, false, false); blockchain.getCheckpointList().init_targets(true, ""); - - // Add checkpoint target at height 5 - // The target system stores targets as m_targets[height+1] = hash - // So to have a checkpoint at height 5, we add target at height 5 - blockchain.getCheckpointList().add_checkpoint_target_for_test(5, "0000000000000000000000000000000000000000000000000000000000000000"); - - // Create checkpoint list with 6 elements (heights 0-5) to match the target - std::vector checkpoint_list(6); // heights 0-5 - crypto::Hash zero_hash = NULL_HASH; - std::fill(checkpoint_list.begin(), checkpoint_list.end(), zero_hash); - - // Calculate hash of the list to match the target - crypto::Hash list_hash = crypto::cn_fast_hash(checkpoint_list.data(), checkpoint_list.size() * sizeof(crypto::Hash)); - - // Set the checkpoint list (this validates against the target) - // Note: This will fail validation unless we use the correct hash - // For testing, we'll just verify the target is set correctly - uint32_t greatest = blockchain.getCheckpointList().get_greatest_target_height(); - ASSERT_EQ(greatest, 5); + // Reset to a clean slate so only our test targets exist + blockchain.getCheckpointList().clear_targets_for_test(); + + // Build a vector of 6 zero hashes (heights 0-5), mirroring old checkpoint.dat + std::vector checkpoint_list(6, NULL_HASH); + crypto::Hash list_hash = crypto::cn_fast_hash( + checkpoint_list.data(), checkpoint_list.size() * sizeof(crypto::Hash)); + + // Add checkpoint target at height 5 using the real list hash + blockchain.getCheckpointList().add_checkpoint_target_for_test(5, common::podToHex(list_hash)); + + // Load the checkpoint list into m_points (mirrors loading old checkpoint.dat) + ASSERT_TRUE(blockchain.getCheckpointList().set_checkpoint_list(std::move(checkpoint_list))); + + ASSERT_EQ(blockchain.getCheckpointList().get_greatest_target_height(), 5u); ASSERT_FALSE(blockchain.is_alternative_block_allowed(0, 0)); @@ -229,14 +225,24 @@ TEST(checkpoints_is_alternative_block_allowed, handles_two_and_more_checkpoints) Blockchain blockchain(currency, tx_pool, logger, false, false); blockchain.getCheckpointList().init_targets(true, ""); - - // Add checkpoint targets at heights 5 and 9 - blockchain.getCheckpointList().add_checkpoint_target_for_test(5, "0000000000000000000000000000000000000000000000000000000000000000"); - blockchain.getCheckpointList().add_checkpoint_target_for_test(9, "0000000000000000000000000000000000000000000000000000000000000000"); - - // Greatest target should be 9 - uint32_t greatest = blockchain.getCheckpointList().get_greatest_target_height(); - ASSERT_EQ(greatest, 9); + // Reset to a clean slate so only our test targets exist + blockchain.getCheckpointList().clear_targets_for_test(); + + // Build vectors of zero hashes mirroring old checkpoint.dat for each checkpoint + // Checkpoint at height 5: 6 hashes (heights 0-5) + std::vector list_5(6, NULL_HASH); + crypto::Hash hash_5 = crypto::cn_fast_hash(list_5.data(), list_5.size() * sizeof(crypto::Hash)); + blockchain.getCheckpointList().add_checkpoint_target_for_test(5, common::podToHex(hash_5)); + + // Checkpoint at height 9: 10 hashes (heights 0-9) + std::vector list_9(10, NULL_HASH); + crypto::Hash hash_9 = crypto::cn_fast_hash(list_9.data(), list_9.size() * sizeof(crypto::Hash)); + blockchain.getCheckpointList().add_checkpoint_target_for_test(9, common::podToHex(hash_9)); + + // Load the full checkpoint list into m_points (mirrors loading final checkpoint.dat) + ASSERT_TRUE(blockchain.getCheckpointList().set_checkpoint_list(std::move(list_9))); + + ASSERT_EQ(blockchain.getCheckpointList().get_greatest_target_height(), 9u); ASSERT_FALSE(blockchain.is_alternative_block_allowed(0, 0)); From d8ff53ef9bc39b90fa05676c087fce02fe20898b Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 28 Apr 2026 19:33:23 -0400 Subject: [PATCH 35/56] add spinner during checkpoint conversion remove confusing log message --- src/CryptoNoteCore/CheckpointsList.cpp | 32 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index e7c56219e..b5fbdb790 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -264,7 +264,18 @@ namespace cn { uint32_t converted_count = 0; uint32_t skipped_count = 0; - + + // Pre-count how many checkpoints are within range so the spinner can show X/N + uint32_t convertible_count = 0; + for (const auto& t : old_targets) + { + if ((t.first - 1) <= max_height) + convertible_count++; + } + + static const char spinner_chars[] = {'|', '/', '-', '\\'}; + uint32_t spin_idx = 0; + // For each old checkpoint, compute the list hash from genesis to that height for (const auto& old_target : old_targets) { @@ -277,6 +288,11 @@ namespace cn { skipped_count++; continue; } + + std::fprintf(stderr, "\r [%c] Converting checkpoint %u/%u (height %u)... ", + spinner_chars[spin_idx % 4], converted_count + 1, convertible_count, height); + std::fflush(stderr); + ++spin_idx; // Get block IDs from genesis (0) to this height std::vector blockIds = getBlockIdsFunc(0, size); @@ -323,6 +339,10 @@ namespace cn { << " (old hash: " << old_target.second << ") to list hash: " << listHash; } + // Clear the spinner line before the final log message + std::fprintf(stderr, "\r%60s\r", ""); + std::fflush(stderr); + logger(INFO) << "Converted " << converted_count << " checkpoint validation targets for P2P compatibility" << " (preserved " << m_old_checkpoint_hashes.size() @@ -580,13 +600,13 @@ namespace cn { logger(INFO) << "Computed chunk " << chunk_index << " hash (blocks " << chunk_start_height << "-" << chunk_end_height << ")" - << " - stored in memory, NOT yet saved to checkpoint.dat" - << " (requires peer consensus with peers having uptime > " - << get_min_peer_uptime_blocks() << " blocks)"; + << " - stored in memory"; } - // NOTE: We do NOT save to checkpoint.dat here - // The chunk will be saved after peer consensus via add_verified_chunk_to_file() + // NOTE: We do NOT save to checkpoint.dat here. + // For chunks within the hardcoded-checkpoint range the caller saves immediately via + // add_verified_chunk_to_file(). For chunks beyond that range the caller waits for + // peer consensus before calling add_verified_chunk_to_file(). return true; } From a3e63e4629350e1357580c4c5dda580198e7d734 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 28 Apr 2026 19:51:09 -0400 Subject: [PATCH 36/56] improve to avoid recompute of chunks hashes --- src/CryptoNoteCore/Blockchain.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 61705075a..a13d0ceb5 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -624,7 +624,7 @@ namespace cn // Chunks are validated during generation, so if they exist and cover the range, we're good bool needs_conversion = (currentHeight > 0) && (greatestTargetHeight > 0) && - (currentCoveredHeight < greatestTargetHeight); + (currentCoveredHeight < std::min(greatestTargetHeight, currentHeight)); if (needs_conversion) { @@ -640,10 +640,11 @@ namespace cn m_checkpoints.convert_old_checkpoints_to_list_hashes(getBlockIdsFunc, currentHeight); } - else if (currentCoveredHeight >= greatestTargetHeight && greatestTargetHeight > 0) + else if (greatestTargetHeight > 0) { - logger(DEBUGGING) << "Skipping target conversion - valid chunks already cover hardcoded checkpoint range (up to height " - << greatestTargetHeight << ")"; + logger(DEBUGGING) << "Skipping target conversion - chunks cover up to height " + << currentCoveredHeight << " (effective target is " + << std::min(greatestTargetHeight, currentHeight) << ")"; } } catch (const std::exception& e) From 71d525d74b8fb4e63fc7b221b6b84ef1cabfce09 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 28 Apr 2026 21:38:33 -0400 Subject: [PATCH 37/56] refine checkpoint conversion * at init make sure conversion from old format to new format is only done once, avoiding edge cases where we would try to convert new format. --- src/CryptoNoteCore/Blockchain.cpp | 91 ++++++++++++++++++-------- src/CryptoNoteCore/CheckpointList.h | 11 +++- src/CryptoNoteCore/CheckpointsList.cpp | 25 +++++++ 3 files changed, 95 insertions(+), 32 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index a13d0ceb5..0f2e44dfc 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -485,7 +485,11 @@ namespace cn m_config_folder = config_folder; m_checkpoints.init_targets(testnet, appendPath(config_folder, m_currency.checkpointFileName())); - m_checkpoints.load_checkpoints_from_file(); + // Only load checkpoint.dat if it is the new chunk-based format. + // Old-style files (first entry = genesis block hash) are ignored here; + // they are handled once in the try block below after the blockchain is loaded. + if (!m_checkpoints.is_old_style_checkpoint_file()) + m_checkpoints.load_checkpoints_from_file(); if (!m_blocks.open(appendPath(config_folder, m_currency.blocksFileName()), appendPath(config_folder, m_currency.blockIndexesFileName()), 1024)) @@ -619,38 +623,67 @@ namespace cn uint32_t currentHeight = static_cast(m_blocks.size() - 1); uint32_t greatestTargetHeight = m_checkpoints.get_greatest_target_height(); uint32_t currentCoveredHeight = m_checkpoints.get_covered_height(); - - // Only convert if we don't have chunks covering the hardcoded checkpoint range - // Chunks are validated during generation, so if they exist and cover the range, we're good - bool needs_conversion = (currentHeight > 0) && - (greatestTargetHeight > 0) && - (currentCoveredHeight < std::min(greatestTargetHeight, currentHeight)); - - if (needs_conversion) - { - logger(DEBUGGING) << "Converting checkpoint validation targets for P2P compatibility (chunks cover up to " - << currentCoveredHeight << ", last checkpoint at " << greatestTargetHeight << ")"; - - // Convert old checkpoints (individual block hashes) to new format (list hashes) - // This is only needed if we're going to use old-style full checkpoint lists - // Only convert checkpoints up to current blockchain height to avoid warnings for unsynced blocks - auto getBlockIdsFunc = [this](uint32_t startHeight, uint32_t maxCount) -> std::vector { - return m_blockIndex.getBlockIds(startHeight, maxCount); - }; - - m_checkpoints.convert_old_checkpoints_to_list_hashes(getBlockIdsFunc, currentHeight); - } - else if (greatestTargetHeight > 0) + uint32_t computeUpTo = std::min(greatestTargetHeight, currentHeight); + + auto getBlockIdsFunc = [this](uint32_t startHeight, uint32_t maxCount) -> std::vector { + return m_blockIndex.getBlockIds(startHeight, maxCount); + }; + + if (currentHeight > 0 && greatestTargetHeight > 0 && computeUpTo > 0) { - logger(DEBUGGING) << "Skipping target conversion - chunks cover up to height " - << currentCoveredHeight << " (effective target is " - << std::min(greatestTargetHeight, currentHeight) << ")"; + uint32_t chunk_size = m_checkpoints.get_chunk_size(); + uint32_t last_chunk = (computeUpTo - 1) / chunk_size; + + if (m_checkpoints.is_old_style_checkpoint_file()) + { + // ── ONE-TIME TRANSITION ────────────────────────────────────────────── + // Convert m_targets to cumulative list hashes up to what we actually have, + // then rebuild all chunks from scratch and write the new-format file. + // Writing chunk 0 overwrites the old file — transition never repeats. + logger(INFO) << "Old-style checkpoint.dat: one-time transition to chunk format " + << "(up to height " << computeUpTo << ")"; + + m_checkpoints.convert_old_checkpoints_to_list_hashes(getBlockIdsFunc, computeUpTo); + + for (uint32_t idx = 0; idx <= last_chunk; ++idx) + { + uint32_t chunk_start = idx * chunk_size + 1; + uint32_t chunk_end = (idx + 1) * chunk_size; + if (currentHeight < chunk_end) break; + if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start)) + { logger(WARNING) << "Transition: chunk " << idx << " failed validation — stopping"; break; } + if (!m_checkpoints.add_verified_chunk_to_file(idx)) + logger(WARNING) << "Transition: chunk " << idx << " could not be written"; + } + + logger(INFO) << "Transition complete — checkpoint.dat is now chunk-based"; + } + else if (currentCoveredHeight < computeUpTo) + { + // ── NORMAL PATH ────────────────────────────────────────────────────── + // New-style or absent: append only the missing chunks up to computeUpTo. + uint32_t first_chunk = m_checkpoints.get_chunk_count(); + + logger(INFO) << "Appending checkpoint chunks " << first_chunk << ".." << last_chunk + << " (heights " << (currentCoveredHeight + 1) << ".." << computeUpTo << ")"; + + for (uint32_t idx = first_chunk; idx <= last_chunk; ++idx) + { + uint32_t chunk_start = idx * chunk_size + 1; + uint32_t chunk_end = (idx + 1) * chunk_size; + if (currentHeight < chunk_end) + { logger(DEBUGGING) << "Chunk " << idx << " incomplete — will finish during sync"; break; } + if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start)) + { logger(WARNING) << "Chunk " << idx << " failed validation — stopping"; break; } + if (!m_checkpoints.add_verified_chunk_to_file(idx)) + logger(WARNING) << "Chunk " << idx << " could not be appended to checkpoint.dat"; + } + } } } catch (const std::exception& e) { - logger(WARNING, BRIGHT_YELLOW) << "Error converting checkpoints: " << e.what(); - // Continue - old checkpoints will still work for individual block validation + logger(WARNING, BRIGHT_YELLOW) << "Error building checkpoint chunks: " << e.what(); } // Generate checkpoint.dat from local blockchain if it doesn't exist or is incomplete @@ -670,7 +703,7 @@ namespace cn // generate them from the local blockchain if (currentHeight > 0 && greatestTargetHeight > 0) { - // Calculate target height: up to greatest target, but not more than current height + // Calculate target height: up to greatest known target, but not more than current height uint32_t targetHeight = std::min(greatestTargetHeight, currentHeight); if (currentCoveredHeight < targetHeight) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index bae9380fa..982889634 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -30,6 +30,11 @@ namespace cn void init_targets(bool is_testnet, const std::string& save_file); + // Returns true if checkpoint.dat exists and its first entry is the genesis block hash, + // which identifies the old per-block format. New-style files start with a chunk hash + // (hash of blocks 1..chunk_size) which can never equal the genesis block hash. + bool is_old_style_checkpoint_file() const; + // Convert old-style checkpoints (individual block hashes) to new-style (list hashes) // This allows the same checkpoint data in CryptoNoteConfig.h to work with both systems // getBlockIdsFunc: function that returns block IDs from genesis (0) to the specified height @@ -314,9 +319,9 @@ namespace cn uint32_t get_greatest_target_height() const { - if (m_targets.empty()) - return 0; - return m_targets.rbegin()->first - 1; + uint32_t hardcoded = m_targets.empty() ? 0 : m_targets.rbegin()->first - 1; + uint32_t dns = m_dns_checkpoint_hashes.empty() ? 0 : m_dns_checkpoint_hashes.rbegin()->first; + return hardcoded > dns ? hardcoded : dns; } bool is_ready() const diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index b5fbdb790..2e5cf200e 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -242,10 +242,35 @@ namespace cn { } } + bool CheckpointList::is_old_style_checkpoint_file() const + { + auto genesis_it = m_old_checkpoint_hashes.find(0); + if (genesis_it == m_old_checkpoint_hashes.end()) + return false; // no genesis checkpoint to compare against + + std::ifstream file(m_save_file, std::ios::binary); + if (!file.is_open()) + return false; // file absent → not old-style + + crypto::Hash first_hash; + if (!file.read(reinterpret_cast(&first_hash), sizeof(first_hash))) + return false; + + return first_hash == genesis_it->second; + } + void CheckpointList::convert_old_checkpoints_to_list_hashes( std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc, uint32_t max_height) { + // If chunks already cover max_height, checkpoint.dat is already new-style — + // no need to convert m_targets (and we must not treat chunk hashes as block hashes). + if (get_covered_height() >= max_height) + { + logger(DEBUGGING) << "Skipping conversion - chunks already cover height " << max_height; + return; + } + // Store old targets temporarily (they're individual block hashes from CryptoNoteConfig.h) // Note: m_targets stores size (height+1) as key, so we need to convert back to get heights // IMPORTANT: m_old_checkpoint_hashes is preserved - it contains the individual block hashes From 1c5f5982623e4eb5ea3926cb64070120ae4387f6 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 12:05:57 -0400 Subject: [PATCH 38/56] fix: build errors * windows: logging fixes * mingw: build fixes phmap.h improv: * At cache rebuild: verify chain linkage between blocks --- CMakeLists.txt | 3 ++- external/parallel_hashmap/phmap.h | 2 +- src/CryptoNoteCore/Blockchain.cpp | 15 +++++++++++++++ src/CryptoNoteCore/CheckpointList.h | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ac2b6ed9..753c037f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,7 +112,8 @@ else() endif() if(MINGW) set(WARNINGS "${WARNINGS} -Wno-error=unused-value") - set(MINGW_FLAG "-DWIN32_LEAN_AND_MEAN") + # Boost.Bind: avoid deprecated global placeholders (_1, _2) pragma on newer Boost + set(MINGW_FLAG "-DWIN32_LEAN_AND_MEAN -DBOOST_BIND_GLOBAL_PLACEHOLDERS") include_directories(SYSTEM src/platform/mingw) else() set(MINGW_FLAG "") diff --git a/external/parallel_hashmap/phmap.h b/external/parallel_hashmap/phmap.h index 37f0f7c14..194e4f600 100644 --- a/external/parallel_hashmap/phmap.h +++ b/external/parallel_hashmap/phmap.h @@ -2753,7 +2753,7 @@ class parallel_hash_set std::is_nothrow_default_constructible::value&& std::is_nothrow_default_constructible::value) {} -#if (__cplusplus >= 201703L || _MSVC_LANG >= 201402) && (defined(_MSC_VER) || defined(__clang__) || (defined(__GNUC__) && __GNUC__ > 6)) +#if (__cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402)) && (defined(_MSC_VER) || defined(__clang__) || (defined(__GNUC__) && __GNUC__ > 6)) explicit parallel_hash_set(size_t bucket_cnt, const hasher& hash_param = hasher(), const key_equal& eq = key_equal(), diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 0f2e44dfc..440b23821 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -1240,6 +1240,7 @@ namespace cn m_spent_keys.clear(); m_outputs.clear(); m_multisignatureOutputs.clear(); + crypto::Hash prevBlockHash = NULL_HASH; for (uint32_t b = 0; b < m_blocks.size(); ++b) { if (b % 1000 == 0) @@ -1249,6 +1250,20 @@ namespace cn const BlockEntry &block = m_blocks[b]; crypto::Hash blockHash = get_block_hash(block.bl); + + // Verify chain linkage: every block except genesis must reference the previous block's hash. + // A mismatch means blockchain.dat was tampered or is corrupt at this height. + if (b > 0 && block.bl.previousBlockHash != prevBlockHash) + { + logger(ERROR, BRIGHT_RED) + << "Chain linkage broken at height " << b + << ": block.previousBlockHash=" << block.bl.previousBlockHash + << " expected=" << prevBlockHash + << ". blockchain.dat is corrupt or has been tampered."; + return false; + } + prevBlockHash = blockHash; + m_blockIndex.push(blockHash); uint64_t interest = 0; for (uint32_t t = 0; t < block.transactions.size(); ++t) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index 982889634..b84694d0d 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -375,7 +375,7 @@ namespace cn if (hv != expected_hash) { // Block hash doesn't match hardcoded checkpoint - validation failed - logger(logging::ERROR) << "<< CheckpointList.cpp << " << "Checkpoint validation FAILED at height " + logger(logging::ERROR) << "CheckpointList.cpp: Checkpoint validation FAILED at height " << height << "! Expected (from CryptoNoteConfig.h): " << expected_hash << ", Got (from block): " << hv; return is_in_zone_failed; From dd66fc346cdb00ada62c4baff4846be1d33dd1d2 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 12:56:03 -0400 Subject: [PATCH 39/56] fix build errors * windows: undef ERROR in CheckpointList.h * MinGW: main.cpp, use static char[] for service name --- src/CryptoNoteCore/CheckpointList.h | 3 +++ src/PaymentGateService/main.cpp | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index b84694d0d..4790ac300 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -15,6 +15,9 @@ #include #include "../CryptoNoteConfig.h" // For P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS constants +#ifdef _WIN32 +#undef ERROR +#endif namespace cn { class CheckpointList diff --git a/src/PaymentGateService/main.cpp b/src/PaymentGateService/main.cpp index 6307e6fc8..e0bd308c6 100644 --- a/src/PaymentGateService/main.cpp +++ b/src/PaymentGateService/main.cpp @@ -123,8 +123,10 @@ int daemonize() { int runDaemon() { #ifdef _WIN32 + // Mutable buffer: SERVICE_TABLE_ENTRY expects LPSTR (non-const); string literals are const. + static char serviceDisplayName[] = "Payment Gate"; SERVICE_TABLE_ENTRY serviceTable[] { - { "Payment Gate", serviceMain }, + { serviceDisplayName, serviceMain }, { NULL, NULL } }; From 4a2349158c1c588202360ff170482507515ff319 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 15:40:44 -0400 Subject: [PATCH 40/56] fix to type size_t for peer count in ChunkValidationManager * in order to fix macOs build errors --- src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp | 2 +- src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index df595932c..914108b48 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -44,7 +44,7 @@ ChunkValidationManager::ChunkValidationManager(ICore& core, IP2pEndpoint* p2p, const Currency& currency, logging::ILogger& log, - std::atomic& peersCount, + std::atomic& peersCount, std::atomic& stop) : m_core(core) , m_p2p(p2p) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h index 668ec3008..7fbdd143a 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h @@ -12,6 +12,7 @@ #include #include #include +#include #include "CryptoNoteCore/CryptoNoteBasic.h" #include "CryptoNoteCore/CheckpointList.h" @@ -36,7 +37,7 @@ namespace cn IP2pEndpoint* p2p, const Currency& currency, logging::ILogger& log, - std::atomic& peersCount, + std::atomic& peersCount, std::atomic& stop); // Update P2P endpoint (called when endpoint is set/updated) @@ -90,7 +91,7 @@ namespace cn IP2pEndpoint* m_p2p; const Currency& m_currency; logging::LoggerRef logger; - std::atomic& m_peersCount; + std::atomic& m_peersCount; std::atomic& m_stop; // Pending chunk hash responses: (peer_id, chunk_index) -> (hash, timestamp) From 6a47b64f23d7ee47ad1846ef5e88ba74604cc0bb Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 16:38:32 -0400 Subject: [PATCH 41/56] attempt to fix macOs boost warning. * new boost version, probably 1.90.0 and base on our use should not break...TBC --- .github/workflows/check.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 11b5d1bb6..2a0715b2e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -409,14 +409,15 @@ jobs: - name: Install dependencies run: | - brew install boost@1.85 + brew install boost + brew list --versions boost - name: Build id: build run: | mkdir build cd build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DSTATIC=ON -DBOOST_ROOT=/usr/local/opt/boost@1.85 + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DSTATIC=ON -DBOOST_ROOT="$(brew --prefix boost) make -j2 - name: Prepare release From 83e656a41029dcdfe76596ed14b979ec32067de2 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 16:55:16 -0400 Subject: [PATCH 42/56] address nodejs 20 -> 24 warnings --- .github/workflows/check.yml | 16 ++++++++-------- .github/workflows/windows.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2a0715b2e..fd7a54c7c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -30,10 +30,10 @@ jobs: echo "release_name=${release_name}" >> $env:GITHUB_OUTPUT - name: Install msbuild - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@v3 - name: Restore Boost - uses: actions/cache@v4 + uses: actions/cache@v5 id: restore-boost with: path: ${{env.BOOST_ROOT}} @@ -67,7 +67,7 @@ jobs: cp build/tests/Release/*_tests.exe build/conceal - name: Upload To GH Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ steps.setup.outputs.release_name }} path: build/conceal @@ -151,7 +151,7 @@ jobs: cp build/tests/*_tests.exe build/conceal - name: Upload To GH Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ steps.setup.outputs.release_name }} path: build/conceal @@ -220,7 +220,7 @@ jobs: cp build/tests/*_tests build/conceal - name: Upload To GH Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ steps.setup.outputs.release_name }} path: build/conceal @@ -290,7 +290,7 @@ jobs: cp build/tests/*_tests build/conceal - name: Upload To GH Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ steps.setup.outputs.release_name }} path: build/conceal @@ -360,7 +360,7 @@ jobs: cp build/tests/*_tests build/conceal - name: Upload To GH Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ steps.setup.outputs.release_name }} path: build/conceal @@ -430,7 +430,7 @@ jobs: cp build/tests/*_tests build/conceal - name: Upload To GH Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ steps.setup.outputs.release_name }} path: build/conceal diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index afef10d13..aa6847dd9 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v5 - name: Setup msbuild - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@v3 - name: Restore Boost uses: actions/cache@v4 From cafffa068ae182ea87857026a9b4c2ff6754fc8e Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 17:15:34 -0400 Subject: [PATCH 43/56] fix typo in macOs cmake command --- .github/workflows/check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index fd7a54c7c..d533d93dc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -417,7 +417,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DSTATIC=ON -DBOOST_ROOT="$(brew --prefix boost) + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=ON -DSTATIC=ON -DBOOST_ROOT="$(brew --prefix boost)" make -j2 - name: Prepare release From ea8201b8e3219243b3851bae7e5be37b74ba8eaf Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 17:26:38 -0400 Subject: [PATCH 44/56] tweak CMakeLists for macOS , boost configuration. --- CMakeLists.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 753c037f1..abaa02ab5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,6 +145,10 @@ if(APPLE) add_definitions(/DHAVE_ROTR) endif() +if(POLICY CMP0167) + cmake_policy(SET CMP0167 OLD) +endif() + if(STATIC) set(Boost_NO_BOOST_CMAKE ON) set(Boost_USE_STATIC_LIBS ON) @@ -152,7 +156,17 @@ if(STATIC) endif() #set(Boost_DEBUG on) -find_package(Boost 1.55 REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options) +set(BOOST_COMPONENTS filesystem thread date_time chrono regex serialization program_options) + +if(APPLE) + find_package(Boost 1.55 COMPONENTS system ${BOOST_COMPONENTS}) + if(NOT Boost_FOUND) + message(STATUS "Boost.System library not found; retrying without system component") + find_package(Boost 1.55 REQUIRED COMPONENTS ${BOOST_COMPONENTS}) + endif() +else() + find_package(Boost 1.55 REQUIRED COMPONENTS system ${BOOST_COMPONENTS}) +endif() include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) if(MINGW) From 22711606ef234bd9d7b4dd6aa600ede921d69906 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 17:41:03 -0400 Subject: [PATCH 45/56] tweak InProcessNode.h, InProcessNode.cpp for compatibility with boost 1.90.0 for macOS build --- src/InProcessNode/InProcessNode.cpp | 6 +++++- src/InProcessNode/InProcessNode.h | 13 +++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/InProcessNode/InProcessNode.cpp b/src/InProcessNode/InProcessNode.cpp index e72be2730..a955868dc 100644 --- a/src/InProcessNode/InProcessNode.cpp +++ b/src/InProcessNode/InProcessNode.cpp @@ -63,7 +63,11 @@ void InProcessNode::init(const Callback& callback) { protocol.addObserver(this); core.addObserver(this); - work.reset(new boost::asio::io_service::work(ioService)); +#if BOOST_VERSION >= 106600 + work.reset(new InProcessNodeWork(ioService.get_executor())); +#else + work.reset(new InProcessNodeWork(ioService)); +#endif workerThread.reset(new std::thread(&InProcessNode::workerFunc, this)); state = INITIALIZED; diff --git a/src/InProcessNode/InProcessNode.h b/src/InProcessNode/InProcessNode.h index a9b926875..1c7bd6ab7 100644 --- a/src/InProcessNode/InProcessNode.h +++ b/src/InProcessNode/InProcessNode.h @@ -18,11 +18,20 @@ #include #include +#include namespace cn { class core; +#if BOOST_VERSION >= 106600 +using InProcessNodeIoContext = boost::asio::io_context; +using InProcessNodeWork = boost::asio::executor_work_guard; +#else +using InProcessNodeIoContext = boost::asio::io_service; +using InProcessNodeWork = boost::asio::io_service::work; +#endif + class InProcessNode : public INode, public cn::ICryptoNoteProtocolObserver, public cn::ICoreObserver { public: InProcessNode(cn::ICore& core, cn::ICryptoNoteProtocolQuery& protocol); @@ -135,9 +144,9 @@ class InProcessNode : public INode, public cn::ICryptoNoteProtocolObserver, publ cn::ICryptoNoteProtocolQuery& protocol; tools::ObserverManager observerManager; - boost::asio::io_service ioService; + InProcessNodeIoContext ioService; std::unique_ptr workerThread; - std::unique_ptr work; + std::unique_ptr work; BlockchainExplorerDataBuilder blockchainExplorerDataBuilder; From 2a5da296962e552ebcf016607e6761736dd97c16 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 18:19:05 -0400 Subject: [PATCH 46/56] address boost io_context per version --- src/InProcessNode/InProcessNode.cpp | 44 +++++++++++++++++------------ src/InProcessNode/InProcessNode.h | 12 +++++++- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/InProcessNode/InProcessNode.cpp b/src/InProcessNode/InProcessNode.cpp index a955868dc..f1de35903 100644 --- a/src/InProcessNode/InProcessNode.cpp +++ b/src/InProcessNode/InProcessNode.cpp @@ -63,7 +63,7 @@ void InProcessNode::init(const Callback& callback) { protocol.addObserver(this); core.addObserver(this); -#if BOOST_VERSION >= 106600 +#if BOOST_VERSION >= 109000 work.reset(new InProcessNodeWork(ioService.get_executor())); #else work.reset(new InProcessNodeWork(ioService)); @@ -73,7 +73,7 @@ void InProcessNode::init(const Callback& callback) { state = INITIALIZED; } - ioService.post(std::bind(callback, ec)); + postIoService(std::bind(callback, ec)); } bool InProcessNode::shutdown() { @@ -93,10 +93,18 @@ bool InProcessNode::doShutdown() { work.reset(); ioService.stop(); workerThread->join(); - ioService.reset(); + resetIoService(); return true; } +void InProcessNode::resetIoService() { +#if BOOST_VERSION >= 109000 + ioService.restart(); +#else + ioService.reset(); +#endif +} + void InProcessNode::workerFunc() { ioService.run(); } @@ -111,7 +119,7 @@ void InProcessNode::getNewBlocks(std::vector&& knownBlockIds, std: return; } - ioService.post( + postIoService( std::bind(&InProcessNode::getNewBlocksAsync, this, std::move(knownBlockIds), @@ -185,7 +193,7 @@ void InProcessNode::getTransactionOutsGlobalIndices(const crypto::Hash& transact return; } - ioService.post( + postIoService( std::bind(&InProcessNode::getTransactionOutsGlobalIndicesAsync, this, std::cref(transactionHash), @@ -235,7 +243,7 @@ void InProcessNode::getRandomOutsByAmounts(std::vector&& amounts, uint return; } - ioService.post( + postIoService( std::bind(&InProcessNode::getRandomOutsByAmountsAsync, this, std::move(amounts), @@ -292,7 +300,7 @@ void InProcessNode::relayTransaction(const cn::Transaction& transaction, const C return; } - ioService.post( + postIoService( std::bind(&InProcessNode::relayTransactionAsync, this, transaction, @@ -461,7 +469,7 @@ void InProcessNode::queryBlocks(std::vector&& knownBlockIds, uint6 return; } - ioService.post( + postIoService( std::bind(&InProcessNode::queryBlocksLiteAsync, this, std::move(knownBlockIds), @@ -523,7 +531,7 @@ void InProcessNode::getPoolSymmetricDifference(std::vector&& known return; } - ioService.post([this, knownPoolTxIds, knownBlockId, &isBcActual, &newTxs, &deletedTxIds, callback] () mutable { + postIoService([this, knownPoolTxIds, knownBlockId, &isBcActual, &newTxs, &deletedTxIds, callback] () mutable { this->getPoolSymmetricDifferenceAsync(std::move(knownPoolTxIds), knownBlockId, isBcActual, newTxs, deletedTxIds, callback); }); } @@ -556,7 +564,7 @@ void InProcessNode::getMultisignatureOutputByGlobalIndex(uint64_t amount, uint32 return; } - ioService.post([this, amount, gindex, &out, callback]() mutable { + postIoService([this, amount, gindex, &out, callback]() mutable { this->getOutByMSigGIndexAsync(amount, gindex, out, callback); }); } @@ -581,7 +589,7 @@ void InProcessNode::getBlocks(const std::vector& blockHeights, std::ve return; } - ioService.post( + postIoService( std::bind( static_cast< void(InProcessNode::*)( @@ -665,7 +673,7 @@ void InProcessNode::getBlocks(const std::vector& blockHashes, std: return; } - ioService.post( + postIoService( std::bind( static_cast< void(InProcessNode::*)( @@ -728,7 +736,7 @@ void InProcessNode::getBlocks(uint64_t timestampBegin, uint64_t timestampEnd, ui return; } - ioService.post( + postIoService( std::bind( static_cast< void(InProcessNode::*)( @@ -804,7 +812,7 @@ void InProcessNode::getTransactions(const std::vector& transaction return; } - ioService.post( + postIoService( std::bind( static_cast< void(InProcessNode::*)( @@ -869,7 +877,7 @@ void InProcessNode::getPoolTransactions(uint64_t timestampBegin, uint64_t timest return; } - ioService.post( + postIoService( std::bind( &InProcessNode::getPoolTransactionsAsync, this, @@ -928,7 +936,7 @@ void InProcessNode::getTransactionsByPaymentId(const crypto::Hash& paymentId, st return; } - ioService.post( + postIoService( std::bind( &InProcessNode::getTransactionsByPaymentIdAsync, this, @@ -949,7 +957,7 @@ void InProcessNode::getTransaction(const crypto::Hash &transactionHash, cn::Tran return; } - ioService.post( + postIoService( std::bind( static_cast< void (InProcessNode::*)( @@ -1044,7 +1052,7 @@ void InProcessNode::isSynchronized(bool& syncStatus, const Callback& callback) { return; } - ioService.post( + postIoService( std::bind( &InProcessNode::isSynchronizedAsync, this, diff --git a/src/InProcessNode/InProcessNode.h b/src/InProcessNode/InProcessNode.h index 1c7bd6ab7..e8abb6b6f 100644 --- a/src/InProcessNode/InProcessNode.h +++ b/src/InProcessNode/InProcessNode.h @@ -24,7 +24,7 @@ namespace cn { class core; -#if BOOST_VERSION >= 106600 +#if BOOST_VERSION >= 109000 using InProcessNodeIoContext = boost::asio::io_context; using InProcessNodeWork = boost::asio::executor_work_guard; #else @@ -133,6 +133,16 @@ class InProcessNode : public INode, public cn::ICryptoNoteProtocolObserver, publ std::error_code doGetTransaction(const crypto::Hash &transactionHash, cn::Transaction &transaction); void workerFunc(); bool doShutdown(); + void resetIoService(); + + template + void postIoService(Handler handler) { +#if BOOST_VERSION >= 109000 + boost::asio::post(ioService, handler); +#else + ioService.post(handler); +#endif + } enum State { NOT_INITIALIZED, From 0cb2327fba2e41bed5350a039c6273c429c9bb44 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 18:45:54 -0400 Subject: [PATCH 47/56] refine checkpoint strategy --- CHECKPOINT_STRATEGY_REFINED.md | 272 +++++++++++++++------------------ 1 file changed, 125 insertions(+), 147 deletions(-) diff --git a/CHECKPOINT_STRATEGY_REFINED.md b/CHECKPOINT_STRATEGY_REFINED.md index 7e6c7c705..19b2afc1f 100644 --- a/CHECKPOINT_STRATEGY_REFINED.md +++ b/CHECKPOINT_STRATEGY_REFINED.md @@ -2,173 +2,151 @@ ## Overview -This document describes the refined checkpoint strategy that combines: -1. **CryptoNoteConfig.h** checkpoints (PRIORITY 1 - highest) -2. **DNS_CHECKPOINT_DOMAIN** checkpoints (PRIORITY 2 - secondary) -3. **blockchain.dat** (PRIORITY 3 - fallback) -4. **P2P consensus** (for chunks beyond trusted checkpoints) +This document describes the checkpoint strategy combining four sources in priority order: + +1. **CryptoNoteConfig.h** (PRIORITY 1 — hardcoded, highest trust) +2. **DNS_CHECKPOINT_DOMAIN** (PRIORITY 2 — updatable without release) +3. **blockchain.dat** (PRIORITY 3 — local fallback) +4. **P2P consensus** (for chunks beyond all trusted checkpoints) + +`get_greatest_target_height()` returns `max(hardcoded_max, dns_max)` and is the canonical +ceiling used everywhere. + +--- + +## checkpoint.dat Formats + +| Format | Description | Detection | +|--------|-------------|-----------| +| **Old-style** | One hash per individual block (~60 MB) | First entry == genesis block hash | +| **New-style (chunks)** | One hash per `chunk_size` blocks (~6 KB) | First entry != genesis block hash | + +`is_old_style_checkpoint_file()` reads only the first 32 bytes and compares against +`m_old_checkpoint_hashes[0]` (genesis hash from `CryptoNoteConfig.h`). + +--- + +## Startup Init Flow + +``` +init_targets() ← populate hardcoded + DNS hashes from CryptoNoteConfig.h + +is_old_style_checkpoint_file()? + YES → skip load (file will be overwritten later) + NO → load_checkpoints_from_file() (fast, ~6 KB read) + +[blockchain.dat loaded] + +computeUpTo = min(get_greatest_target_height(), currentHeight) + +if old-style file detected: + ── ONE-TIME TRANSITION ────────────────────────────────────────────────────── + convert_old_checkpoints_to_list_hashes(computeUpTo) + └─ updates m_targets with cumulative list hashes (legacy P2P compatibility) + └─ guard: skips if chunks already cover max_height (safety net — should not happen in normal flow) + generate chunks 0 .. (computeUpTo-1)/chunk_size → add_verified_chunk_to_file() + └─ writing chunk 0 truncates the old file → transition never repeats on next restart + +else (new-style or absent): + ── NORMAL PATH ────────────────────────────────────────────────────────────── + append chunks [currentCoveredHeight+1 .. computeUpTo] only + └─ each chunk validated against hardcoded/DNS checkpoint at its boundary + └─ stops immediately if validation fails (blockchain mismatch) +``` + +**One-time transition guarantee:** after the old-style path runs, chunk 0 is written with +a chunk hash (not the genesis block hash), so `is_old_style_checkpoint_file()` returns +`false` on every subsequent restart. + +--- ## Priority Order for Block Hashes in Chunks -When generating chunk hashes, the system uses the following priority order: - -1. **CryptoNoteConfig.h** (PRIORITY 1) - Hardcoded in source code, highest trust -2. **DNS_CHECKPOINT_DOMAIN** (PRIORITY 2) - Fetched from DNS, secondary trusted source -3. **blockchain.dat** (PRIORITY 3) - Actual block hash from local blockchain (fallback) - -**Example:** -- Chunk contains blocks 1690001-1700000 -- Block 1690753 has checkpoint in DNS -- Block 1700000 has checkpoint in CryptoNoteConfig.h -- Result: - - Blocks 1690001-1690752: use blockchain.dat - - Block 1690753: use DNS checkpoint hash - - Blocks 1690754-1699999: use blockchain.dat - - Block 1700000: use CryptoNoteConfig.h checkpoint hash - -## Phase 1: Initial Setup (No checkpoint.dat) - -**When:** Node starts with `blockchain.dat` but no `checkpoint.dat` - -**Process:** -1. Load checkpoints from **CryptoNoteConfig.h** (PRIORITY 1) -2. Fetch checkpoints from **DNS_CHECKPOINT_DOMAIN** (PRIORITY 2) -3. Generate chunks up to the last **CryptoNoteConfig.h** checkpoint: - - Use priority order: CryptoNoteConfig.h > DNS > blockchain.dat - - Validate that blockchain.dat matches checkpoints - - Save chunks to `checkpoint.dat` (auto-confirmed) -4. If DNS checkpoints extend beyond CryptoNoteConfig.h: - - Create chunks up to highest DNS checkpoint - - Use priority order during generation - - Auto-confirm (validated against DNS checkpoints) -5. Beyond DNS checkpoints: - - Create chunks in memory only - - Seek P2P validation before saving to `checkpoint.dat` - -**Result:** `checkpoint.dat` contains chunks validated against CryptoNoteConfig.h and DNS checkpoints. - -## Phase 2: Milestone Chunk Creation (During Sync) - -**When:** Every `chunk_size` blocks (10,000 mainnet, 25,000 testnet) - -**Process:** -1. Compute chunk hash using **priority order**: - - For each block in chunk: - - If checkpoint in CryptoNoteConfig.h → use it - - Else if checkpoint in DNS → use it - - Else → use blockchain.dat hash -2. Store chunk hash **in memory only** (not saved to `checkpoint.dat` yet) -3. Seek **P2P validation**: - - Sample K peers (M must agree) - - Require network diversity (n distinct /16 networks) - - Require peer uptime > minimum (12k blocks mainnet, 28k testnet) -4. If consensus reached: - - Save chunk to `checkpoint.dat` (confirmation) -5. If consensus fails: - - Trigger blockchain rollback - - Truncate `checkpoint.dat` to last valid chunk - -**Key Point:** Priority order ensures trusted checkpoints (CryptoNoteConfig.h, DNS) take precedence over blockchain.dat during chunk generation. - -## Phase 3: Startup Validation - -**When:** Node restarts with existing `checkpoint.dat` - -**Process:** -1. Load all chunks from `checkpoint.dat` (trusted - already validated) -2. Get current blockchain height -3. Find highest checkpoints from: - - CryptoNoteConfig.h (PRIORITY 1) - - DNS (PRIORITY 2) -4. For each checkpoint we can verify (checkpoint height <= current blockchain height): - - Compute chunk hash using **priority order**: - - CryptoNoteConfig.h checkpoints in chunk - - DNS checkpoints in chunk (if not in CryptoNoteConfig.h) - - blockchain.dat for remaining blocks - - Compare with stored chunk hash in `checkpoint.dat` - - If mismatch: - - Rollback blockchain to chunk boundary - - Truncate `checkpoint.dat` to last valid chunk -5. If checkpoint is beyond current height: - - Continue syncing - - Validate when reaching that height via P2P - -**Example:** -- Current blockchain height: 1,800,000 -- CryptoNoteConfig.h highest: 1,700,000 (chunk 170) -- DNS highest: 1,900,000 (chunk 190, can't verify yet - not synced) -- We can only verify chunk 170 (CryptoNoteConfig.h checkpoint) -- Chunk 190 will be validated when we sync to height 1,900,000 - -## Priority Order Implementation - -### Chunk Generation (`generate_chunks_from_block_ids`, `add_chunk_from_block_ids`) - -1. Get block IDs from blockchain.dat for the chunk -2. Build checkpoint map with priority: - - Add CryptoNoteConfig.h checkpoints (PRIORITY 1) - - Add DNS checkpoints (PRIORITY 2, only if not in CryptoNoteConfig.h) -3. For each checkpoint in chunk: - - Validate blockchain.dat matches checkpoint hash - - Replace blockchain.dat hash with checkpoint hash -4. Compute chunk hash from modified block IDs -5. Store chunk hash - -### Startup Validation (`validate_chunks_against_checkpoints`) - -1. Find highest checkpoints from CryptoNoteConfig.h and DNS -2. For each checkpoint we can verify (based on current height): - - Get chunk containing checkpoint - - Compute chunk hash using priority order - - Compare with stored chunk hash - - If mismatch → rollback - -## Benefits - -1. **Trust Hierarchy:** CryptoNoteConfig.h > DNS > blockchain.dat -2. **Emergency Updates:** DNS can provide checkpoints without code release -3. **Autonomous:** P2P consensus for chunks beyond trusted checkpoints -4. **Backward Compatible:** Works with existing CryptoNoteConfig.h checkpoints -5. **Flexible:** Can rely on P2P when DNS unavailable +When building a chunk hash, each block's hash is resolved in priority order: + +1. **CryptoNoteConfig.h** — if a hardcoded checkpoint exists at that height, use it +2. **DNS** — else if a DNS checkpoint exists at that height, use it +3. **blockchain.dat** — otherwise use the locally stored block hash + +**Example** (chunk covers blocks 1 690 001–1 700 000): +- Block 1 690 753 → DNS checkpoint → DNS hash used +- Block 1 700 000 → CryptoNoteConfig.h checkpoint → hardcoded hash used +- All others → blockchain.dat hash + +--- + +## Phase 2: Chunk Creation During Sync + +**Trigger:** every `chunk_size` blocks added to the chain. + +1. Compute chunk hash using priority order above. +2. Store in memory only (not yet in `checkpoint.dat`). +3. Seek **P2P consensus**: + - Sample K peers (require M agreements, n distinct /16 networks, uptime > minimum). +4. Consensus reached → `add_verified_chunk_to_file()` (appends to `checkpoint.dat`). +5. Consensus failed → rollback blockchain, truncate `checkpoint.dat` to last valid chunk. + +--- + +## Phase 3: Restart Validation + +**Trigger:** node restarts with an existing new-style `checkpoint.dat`. + +1. Load all chunks (auto-confirmed — file only ever contains verified chunks). +2. Validate chunks against hardcoded/DNS checkpoints via `validate_chunks_against_checkpoints()`. +3. Mismatch → rollback + truncate. +4. Append any missing chunks up to `computeUpTo` (normal path above). + +--- + +## `convert_old_checkpoints_to_list_hashes` Guards + +The function is skipped if either condition holds: + +| Guard | Reason | +|-------|--------| +| `get_covered_height() >= max_height` | Safety net against accidental calls from other code paths — if chunks already cover the range, skip silently | + +--- ## Chunk Sizes -- **Mainnet:** 10,000 blocks per chunk -- **Testnet:** 25,000 blocks per chunk +| Network | Blocks per chunk | +|---------|-----------------| +| Mainnet | 10 000 | +| Testnet | 25 000 | -## Consensus Requirements +--- -### Mainnet -- M = 3 (minimum agreements) -- K = 5 (peers to sample) -- n = 2 (distinct /16 networks) -- Uptime: > 12,000 blocks (~16.7 days) +## P2P Consensus Requirements -### Testnet -- M = 2 (minimum agreements) -- K = 3 (peers to sample) -- n = 1 (distinct /16 networks) -- Uptime: > 28,000 blocks (~38.9 days) +| Parameter | Mainnet | Testnet | +|-----------|---------|---------| +| M (min agreements) | 3 | 2 | +| K (peers sampled) | 5 | 3 | +| n (distinct /16 networks) | 2 | 1 | +| Min peer uptime | 12 000 blocks (~16.7 days) | 28 000 blocks (~38.9 days) | + +--- ## DNS Checkpoint Format -DNS TXT records should be in format: +DNS TXT records: ``` height:hash ``` - Example: ``` 100000:55cf271a5c97785fb35fea7ed177cb75f47c18688bd86fc01ae66508878029d6 200000:52533de7f1596154c6954530ae8331fe4f92e92d476f097c6d7d20ebab1c2748 ``` -## Summary +--- -This refined strategy provides: -- **Clear priority order** for checkpoint sources -- **Autonomous operation** via P2P consensus -- **Emergency capability** via DNS checkpoints -- **Efficient validation** using chunked checkpoints -- **Backward compatibility** with existing hardcoded checkpoints +## Summary of Key Properties +- **Old-style detection** is exact: first 32 bytes compared to genesis hash — no heuristics. +- **computeUpTo** = `min(max(hardcoded, DNS), localChainHeight)` — never computes beyond what the node has. +- **Transition is one-shot**: old-style file is overwritten in a single startup; no repeat work. +- **Normal path is minimal**: only appends the chunks that are actually missing. +- **Trust hierarchy**: CryptoNoteConfig.h > DNS > blockchain.dat > P2P. From a0d3f7bbb5df27215d7bb10b11d1f741b7b16c27 Mon Sep 17 00:00:00 2001 From: acktarius Date: Wed, 29 Apr 2026 21:40:23 -0400 Subject: [PATCH 48/56] address memory leak --- src/CryptoNoteCore/Currency.cpp | 24 +++++++++++++++++++--- src/Logging/FileLogger.cpp | 8 ++++++++ src/Logging/FileLogger.h | 1 + src/Logging/LoggerManager.cpp | 6 ++++++ src/Logging/LoggerManager.h | 1 + src/Platform/Linux/System/Dispatcher.cpp | 2 ++ src/Platform/OSX/System/Dispatcher.cpp | 2 ++ src/Platform/Windows/System/Dispatcher.cpp | 2 ++ src/System/ContextGroup.cpp | 5 +++++ 9 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/CryptoNoteCore/Currency.cpp b/src/CryptoNoteCore/Currency.cpp index a46051c17..111d5dff6 100644 --- a/src/CryptoNoteCore/Currency.cpp +++ b/src/CryptoNoteCore/Currency.cpp @@ -482,14 +482,32 @@ namespace cn if (amount_out > amount_in) { - // interest shows up in the output of the W/D transactions and W/Ds always have min fee - if (tx.inputs.size() > 0 && !tx.outputs.empty() && amount_out > amount_in + parameters::MINIMUM_FEE) + // Check if this is actually a deposit multisig spend (withdrawal) + bool isDepositWithdrawal = false; + for (const auto &in : tx.inputs) { + if (in.type() == typeid(MultisignatureInput)) + { + const auto &msig = boost::get(in); + if (msig.term != 0) + { + isDepositWithdrawal = true; + break; + } + } + } + + if (isDepositWithdrawal) + { + // Interest shows up in the output of withdrawal transactions + // W/Ds always have min fee fee = parameters::MINIMUM_FEE; - logger(INFO) << "TRIGGERED: Currency.cpp getTransactionFee"; + return true; } else { + // Not a withdrawal — overspend is invalid + logger(ERROR) << "Transaction has outputs > inputs but is not a deposit withdrawal"; return false; } } diff --git a/src/Logging/FileLogger.cpp b/src/Logging/FileLogger.cpp index 8c8a39453..0727937d9 100644 --- a/src/Logging/FileLogger.cpp +++ b/src/Logging/FileLogger.cpp @@ -12,6 +12,14 @@ namespace logging { FileLogger::FileLogger(Level level) : StreamLogger(level) { } +FileLogger::~FileLogger() { + if (fileStream.is_open()) { + stream = nullptr; + fileStream.flush(); + fileStream.close(); + } +} + void FileLogger::init(const std::string& fileName) { fileStream.open(fileName, std::ios::app); StreamLogger::attachToStream(fileStream); diff --git a/src/Logging/FileLogger.h b/src/Logging/FileLogger.h index ec948a94f..6e25e6495 100644 --- a/src/Logging/FileLogger.h +++ b/src/Logging/FileLogger.h @@ -15,6 +15,7 @@ namespace logging { class FileLogger : public StreamLogger { public: FileLogger(Level level = DEBUGGING); + ~FileLogger(); void init(const std::string& filename); private: diff --git a/src/Logging/LoggerManager.cpp b/src/Logging/LoggerManager.cpp index 256a142e5..143243913 100644 --- a/src/Logging/LoggerManager.cpp +++ b/src/Logging/LoggerManager.cpp @@ -17,6 +17,12 @@ using common::JsonValue; LoggerManager::LoggerManager() { } +LoggerManager::~LoggerManager() { + std::unique_lock lock(reconfigureLock); + LoggerGroup::loggers.clear(); + loggers.clear(); +} + void LoggerManager::operator()(const std::string& category, Level level, boost::posix_time::ptime time, const std::string& body) { std::unique_lock lock(reconfigureLock); LoggerGroup::operator()(category, level, time, body); diff --git a/src/Logging/LoggerManager.h b/src/Logging/LoggerManager.h index c86556041..3d6122504 100644 --- a/src/Logging/LoggerManager.h +++ b/src/Logging/LoggerManager.h @@ -18,6 +18,7 @@ namespace logging { class LoggerManager : public LoggerGroup { public: LoggerManager(); + ~LoggerManager(); void configure(const common::JsonValue& val); virtual void operator()(const std::string& category, Level level, boost::posix_time::ptime time, const std::string& body) override; diff --git a/src/Platform/Linux/System/Dispatcher.cpp b/src/Platform/Linux/System/Dispatcher.cpp index 07b6c4e5a..b4b23399b 100644 --- a/src/Platform/Linux/System/Dispatcher.cpp +++ b/src/Platform/Linux/System/Dispatcher.cpp @@ -445,6 +445,8 @@ void Dispatcher::contextProcedure(void* ucontext) { context.procedure(); } catch(std::exception&) { } + context.procedure = nullptr; + context.interruptProcedure = nullptr; if (context.group != nullptr) { if (context.groupPrev != nullptr) { diff --git a/src/Platform/OSX/System/Dispatcher.cpp b/src/Platform/OSX/System/Dispatcher.cpp index 5d82eaa99..919983481 100644 --- a/src/Platform/OSX/System/Dispatcher.cpp +++ b/src/Platform/OSX/System/Dispatcher.cpp @@ -412,6 +412,8 @@ void Dispatcher::contextProcedure(void* ucontext) { context.procedure(); } catch(std::exception&) { } + context.procedure = nullptr; + context.interruptProcedure = nullptr; if (context.group != nullptr) { if (context.groupPrev != nullptr) { diff --git a/src/Platform/Windows/System/Dispatcher.cpp b/src/Platform/Windows/System/Dispatcher.cpp index 9fdb2d4b2..daea8d994 100644 --- a/src/Platform/Windows/System/Dispatcher.cpp +++ b/src/Platform/Windows/System/Dispatcher.cpp @@ -390,6 +390,8 @@ void Dispatcher::contextProcedure() { context.procedure(); } catch (std::exception&) { } + context.procedure = nullptr; + context.interruptProcedure = nullptr; if (context.group != nullptr) { if (context.groupPrev != nullptr) { diff --git a/src/System/ContextGroup.cpp b/src/System/ContextGroup.cpp index 116a4dac0..0154ea663 100644 --- a/src/System/ContextGroup.cpp +++ b/src/System/ContextGroup.cpp @@ -84,6 +84,11 @@ void ContextGroup::wait() { dispatcher->dispatch(); assert(context == dispatcher->getCurrentContext()); } + + for (NativeContext* context = contextGroup.firstContext; context != nullptr; context = context->groupNext) { + context->procedure = nullptr; + context->interruptProcedure = nullptr; + } } } From 4a63c8b8d0e875b9af74dcecb00a99788af287f2 Mon Sep 17 00:00:00 2001 From: acktarius Date: Thu, 30 Apr 2026 09:40:14 -0400 Subject: [PATCH 49/56] update checkpoints mainnet + testnet --- src/CryptoNoteConfig.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index f33a2be2d..fb6b10ca3 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -455,7 +455,17 @@ namespace cn {1920000, "ba318eb902d6a3f7dd4e7116b13c6b1f2cb4e8c9e70acbde3b6a28a1374c6243"}, {1930000, "6f39f0888c14808710e909fb45c864fdf738cf644e432486d8e01b65dbbc2862"}, {1940000, "fe774dd97a5b975c02abb18ca48bf6fd3a8048b597520befe1f261855a7ccf3f"}, - {1950000, "6147f276271c55319cf6fc03e86bcaa75ea0a2aab126095a9a9f8a811eb229e1"} + {1950000, "6147f276271c55319cf6fc03e86bcaa75ea0a2aab126095a9a9f8a811eb229e1"}, + {1960000, "e5b31d500706cbca7e7eaca4b5762413533c826692c632a1c0a1a0df6771f42d"}, + {1970000, "5b8da5814d2d82bd21db65b2b3c78b0f2038deab2ae16eee721193713c9e2e27"}, + {1980000, "a69211b77326d9ff74fe8be81c281e0a44b0c8bbee643cd6d736e186362fd7f5"}, + {1990000, "fe9b87f2e98f544af8b7ac950b7b5a1eebf0a9508ea613c74bd3304433726b30"}, + {2000000, "bf1e1396f4ee1c21a2d574035c6b422d62b8aa3d4b3268111b1b73357ad49740"}, + {2010000, "c4806687630ae3fc1c371889617f0659f2622ba9bcdcee1a0c349dea87ed4f84"}, + {2020000, "ab632a9b914875a2b30bbf5946859c352801450df35e1fdaee8107da3d1df64e"}, + {2030000, "c0b03673083146e028c609498a95e0f22ae61f06f7c961cd1ac5d8ba99c8e3c6"}, + {2040000, "f6abea5fb93bb391f24599252656d40c1d716cb18a245b200f6b70d962bf2bdf"}, + {2050000, "ab2df8795e771bb9b74e8a893f91673f9f44321844ed11d31e11270c09c6e6ae"} }; const std::initializer_list TESTNET_CHECKPOINTS = { @@ -498,7 +508,13 @@ namespace cn {900000, "a70b6df1794a6d91071cd5fc87719769bf09610d520c2c2134f53908d1e3de40"}, {925000, "a00b47f3610cfd5c509183322fca89e388c0427601ce4db7e397acbcab5a3ee6"}, {950000, "387573b7b9bdbc1d79c28156cf15d7e08ddf248a0257b0ee7ef2731c5c7a0534"}, - {975000, "2bbf6d2fecb329d9c34968e60b0d2814a2d5a2f69ee872b77d152ba795e881ae"} + {975000, "2bbf6d2fecb329d9c34968e60b0d2814a2d5a2f69ee872b77d152ba795e881ae"}, + {1000000, "c2627d61bd357fe38398e6fbc14c5c8510aac12934fc696bf7d7935a762f03a1"}, + {1025000, "325a7b27a91d221c58597e17a913b5bf6c79704c1b45a7849ba5d745c6f82e04"}, + {1050000, "ed30311a19f916184da2e961b7cf527114954dc954817804cc49c5f010632dea"}, + {1075000, "0da0f9185f7f32e9ef29eaebe290e1614cb85987e4fb6ba027570be3d2cc33ac"}, + {1100000, "cd87368b14488a85c2a1064ccf61855db4664919c8c2ddef1c6f088951700f19"}, + {1125000, "04c19d19ed31300a088b9485ddea0a2738821a7ea7da42f8324c8144f4e8c9c3"} }; } // namespace cn From b0cf8b53561571fd83b9d87137bcda6062e84177 Mon Sep 17 00:00:00 2001 From: "nullcrypto (Jay)" <155117721+nullcryptodev@users.noreply.github.com> Date: Tue, 28 Apr 2026 04:31:23 +0100 Subject: [PATCH 50/56] Improve wallet reset success messaging Update success message for wallet reset process. --- src/ConcealWallet/ConcealWallet.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ConcealWallet/ConcealWallet.cpp b/src/ConcealWallet/ConcealWallet.cpp index 7e0fec321..4378031ce 100644 --- a/src/ConcealWallet/ConcealWallet.cpp +++ b/src/ConcealWallet/ConcealWallet.cpp @@ -770,12 +770,16 @@ bool conceal_wallet::reset(const std::vector &) m_wallet->reset(0); m_wallet->addObserver(this); - success_msg_writer(true) << "Reset completed successfully."; + success_msg_writer(true) << "Attempting wallet reset... this could take some time."; std::unique_lock lock(m_walletSynchronizedMutex); m_walletSynchronizedCV.wait(lock, [this] { return m_walletSynchronized; }); + if (m_walletSynchronized) { + success_msg_writer(true) << "Wallet reset was successful"; + } + std::cout << std::endl; return true; @@ -1833,4 +1837,4 @@ void conceal_wallet::depositUpdated(DepositId depositId) void conceal_wallet::depositsUpdated(const std::vector &depositIds) { // Nothing to do here -} \ No newline at end of file +} From e9f79ec3c8e2603ff701f921c89b88797f1d31a7 Mon Sep 17 00:00:00 2001 From: acktarius Date: Thu, 30 Apr 2026 13:08:34 -0400 Subject: [PATCH 51/56] version bump beta * bump to 6.7.4-beta.1 in order to propose on upstream/development * update contributing flow --- CHECKPOINT_STRATEGY_REFINED.md | 152 --------------------------------- CMakeLists.txt | 2 +- CONTRIBUTING.md | 8 +- src/CryptoNoteConfig.h | 2 +- 4 files changed, 7 insertions(+), 157 deletions(-) delete mode 100644 CHECKPOINT_STRATEGY_REFINED.md diff --git a/CHECKPOINT_STRATEGY_REFINED.md b/CHECKPOINT_STRATEGY_REFINED.md deleted file mode 100644 index 19b2afc1f..000000000 --- a/CHECKPOINT_STRATEGY_REFINED.md +++ /dev/null @@ -1,152 +0,0 @@ -# Refined Checkpoint Strategy (Priority-Based) - -## Overview - -This document describes the checkpoint strategy combining four sources in priority order: - -1. **CryptoNoteConfig.h** (PRIORITY 1 — hardcoded, highest trust) -2. **DNS_CHECKPOINT_DOMAIN** (PRIORITY 2 — updatable without release) -3. **blockchain.dat** (PRIORITY 3 — local fallback) -4. **P2P consensus** (for chunks beyond all trusted checkpoints) - -`get_greatest_target_height()` returns `max(hardcoded_max, dns_max)` and is the canonical -ceiling used everywhere. - ---- - -## checkpoint.dat Formats - -| Format | Description | Detection | -|--------|-------------|-----------| -| **Old-style** | One hash per individual block (~60 MB) | First entry == genesis block hash | -| **New-style (chunks)** | One hash per `chunk_size` blocks (~6 KB) | First entry != genesis block hash | - -`is_old_style_checkpoint_file()` reads only the first 32 bytes and compares against -`m_old_checkpoint_hashes[0]` (genesis hash from `CryptoNoteConfig.h`). - ---- - -## Startup Init Flow - -``` -init_targets() ← populate hardcoded + DNS hashes from CryptoNoteConfig.h - -is_old_style_checkpoint_file()? - YES → skip load (file will be overwritten later) - NO → load_checkpoints_from_file() (fast, ~6 KB read) - -[blockchain.dat loaded] - -computeUpTo = min(get_greatest_target_height(), currentHeight) - -if old-style file detected: - ── ONE-TIME TRANSITION ────────────────────────────────────────────────────── - convert_old_checkpoints_to_list_hashes(computeUpTo) - └─ updates m_targets with cumulative list hashes (legacy P2P compatibility) - └─ guard: skips if chunks already cover max_height (safety net — should not happen in normal flow) - generate chunks 0 .. (computeUpTo-1)/chunk_size → add_verified_chunk_to_file() - └─ writing chunk 0 truncates the old file → transition never repeats on next restart - -else (new-style or absent): - ── NORMAL PATH ────────────────────────────────────────────────────────────── - append chunks [currentCoveredHeight+1 .. computeUpTo] only - └─ each chunk validated against hardcoded/DNS checkpoint at its boundary - └─ stops immediately if validation fails (blockchain mismatch) -``` - -**One-time transition guarantee:** after the old-style path runs, chunk 0 is written with -a chunk hash (not the genesis block hash), so `is_old_style_checkpoint_file()` returns -`false` on every subsequent restart. - ---- - -## Priority Order for Block Hashes in Chunks - -When building a chunk hash, each block's hash is resolved in priority order: - -1. **CryptoNoteConfig.h** — if a hardcoded checkpoint exists at that height, use it -2. **DNS** — else if a DNS checkpoint exists at that height, use it -3. **blockchain.dat** — otherwise use the locally stored block hash - -**Example** (chunk covers blocks 1 690 001–1 700 000): -- Block 1 690 753 → DNS checkpoint → DNS hash used -- Block 1 700 000 → CryptoNoteConfig.h checkpoint → hardcoded hash used -- All others → blockchain.dat hash - ---- - -## Phase 2: Chunk Creation During Sync - -**Trigger:** every `chunk_size` blocks added to the chain. - -1. Compute chunk hash using priority order above. -2. Store in memory only (not yet in `checkpoint.dat`). -3. Seek **P2P consensus**: - - Sample K peers (require M agreements, n distinct /16 networks, uptime > minimum). -4. Consensus reached → `add_verified_chunk_to_file()` (appends to `checkpoint.dat`). -5. Consensus failed → rollback blockchain, truncate `checkpoint.dat` to last valid chunk. - ---- - -## Phase 3: Restart Validation - -**Trigger:** node restarts with an existing new-style `checkpoint.dat`. - -1. Load all chunks (auto-confirmed — file only ever contains verified chunks). -2. Validate chunks against hardcoded/DNS checkpoints via `validate_chunks_against_checkpoints()`. -3. Mismatch → rollback + truncate. -4. Append any missing chunks up to `computeUpTo` (normal path above). - ---- - -## `convert_old_checkpoints_to_list_hashes` Guards - -The function is skipped if either condition holds: - -| Guard | Reason | -|-------|--------| -| `get_covered_height() >= max_height` | Safety net against accidental calls from other code paths — if chunks already cover the range, skip silently | - ---- - -## Chunk Sizes - -| Network | Blocks per chunk | -|---------|-----------------| -| Mainnet | 10 000 | -| Testnet | 25 000 | - ---- - -## P2P Consensus Requirements - -| Parameter | Mainnet | Testnet | -|-----------|---------|---------| -| M (min agreements) | 3 | 2 | -| K (peers sampled) | 5 | 3 | -| n (distinct /16 networks) | 2 | 1 | -| Min peer uptime | 12 000 blocks (~16.7 days) | 28 000 blocks (~38.9 days) | - ---- - -## DNS Checkpoint Format - -DNS TXT records: -``` -height:hash -``` -Example: -``` -100000:55cf271a5c97785fb35fea7ed177cb75f47c18688bd86fc01ae66508878029d6 -200000:52533de7f1596154c6954530ae8331fe4f92e92d476f097c6d7d20ebab1c2748 -``` - ---- - -## Summary of Key Properties - -- **Old-style detection** is exact: first 32 bytes compared to genesis hash — no heuristics. -- **computeUpTo** = `min(max(hardcoded, DNS), localChainHeight)` — never computes beyond what the node has. -- **Transition is one-shot**: old-style file is overwritten in a single startup; no repeat work. -- **Normal path is minimal**: only appends the chunks that are actually missing. -- **Trust hierarchy**: CryptoNoteConfig.h > DNS > blockchain.dat > P2P. diff --git a/CMakeLists.txt b/CMakeLists.txt index abaa02ab5..6b195f9fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.5) include(CheckCXXCompilerFlag) -set(VERSION "6.7.3") +set(VERSION "6.7.4-beta.1") set(VERSION_BUILD_NO "Trebopala") # Packaged from main commits set(COMMIT 1db6e66) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab99bfa93..300b2bc7f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,12 +1,14 @@ Development Process -Developers work in their own trees, then submit pull requests when they think their feature or bug fix is ready. +Contributors should work from their own fork of the repository (not push branches directly to the upstream repo unless they are maintainers with that workflow). When you start a change, create a branch named `/`: use a three-letter identifier derived from your name or handle, then a short topic (often kebab-case words). Examples for someone named John Doe: `jdo/fix`, `doe/dependencies`. + +Open pull requests against the upstream **`development`** branch when you consider your feature or bug fix ready. The patch will be accepted if there is broad consensus that it is a good thing. Developers should expect to rework and resubmit patches if they don't match the project's coding conventions or are controversial. -The master branch is regularly built and tested, but is not guaranteed to be completely stable. Tags are regularly created to indicate new official, stable release versions of Conceal. +The `development` branch is regularly built and tested, but is not guaranteed to be completely stable. Tags are regularly created to indicate new official, stable release versions of Conceal. -Feature branches are created when there are major new features being worked on by several people. +Feature branches on upstream may be created when there are major new features being worked on by several people. From time to time a pull request will become outdated. If this occurs, and the pull is no longer automatically mergeable; a comment on the pull will be used to issue a warning of closure. The pull will be closed 15 days after the warning if action is not taken by the author. Pull requests closed in this manner will have their corresponding issue labeled 'stagnant'. diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index fb6b10ca3..056e50f80 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -1,6 +1,6 @@ // Copyright (c) 2012-2017 The Cryptonote developers // Copyright (c) 2017-2018 The Circle Foundation & Conceal Devs -// Copyright (c) 2018-2023 Conceal Network & Conceal Devs +// Copyright (c) 2018-2026 Conceal Network & Conceal Devs // // // Distributed under the MIT/X11 software license, see the accompanying From 8700261eac2cb82f6898a49455a80d78d1dca0fa Mon Sep 17 00:00:00 2001 From: acktarius Date: Thu, 30 Apr 2026 13:19:04 -0400 Subject: [PATCH 52/56] adjust checkpoint min uptime before concensus check --- src/CryptoNoteConfig.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index 056e50f80..d8b0e3afd 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -210,8 +210,8 @@ namespace cn const uint64_t P2P_DEFAULT_INVOKE_TIMEOUT = 60 * 2 * 1000; // 2 minutes const size_t P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT = 5000; // 5 seconds const size_t P2P_CHECKPOINT_LIST_RE_REQUEST = 300; // 5 minutes - const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS = 12000; // Minimum peer uptime for checkpoint verification (mainnet: 12000 blocks × 2 min = 24000 min ≈ 16.7 days) - const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS_TESTNET = 2; // Minimum peer uptime for checkpoint verification (testnet: 6000 blocks × 2 min = 12000 min ≈ 8.3 days, relaxed for smaller network) + const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS = 20; // Minimum uptime for checkpoint verification (mainnet: 20 blocks × 2 min = 40 min, should be enought to cover 10000 blocks) + const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS_TESTNET = 2; // Minimum uptime for checkpoint verification (testnet: 2 blocks × 2 min = 4 min relaxed for testing) // Checkpoint consensus configuration (Mainnet) // M = minimum agreements required (all M peers must agree) From 9ff8eef8d22e3f2779365112c3d07bc3f5883d87 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 5 May 2026 15:29:41 -0400 Subject: [PATCH 53/56] prefetch chunk hash * store it in memory = allow in check zone * at checkpoit height confirm the prefetched with the locally computed hash, and rollback if needed --- src/CryptoNoteConfig.h | 2 +- src/CryptoNoteCore/Blockchain.cpp | 33 +++- src/CryptoNoteCore/CheckpointList.h | 31 ++- src/CryptoNoteCore/CheckpointsList.cpp | 176 ++++++++++++++--- .../CryptoNoteProtocolHandler.cpp | 13 +- .../CryptoNoteProtocolHandlerChunk.cpp | 178 +++++++++++++++++- .../CryptoNoteProtocolHandlerChunk.h | 13 ++ 7 files changed, 402 insertions(+), 44 deletions(-) diff --git a/src/CryptoNoteConfig.h b/src/CryptoNoteConfig.h index d8b0e3afd..c62ee2dd0 100644 --- a/src/CryptoNoteConfig.h +++ b/src/CryptoNoteConfig.h @@ -210,7 +210,7 @@ namespace cn const uint64_t P2P_DEFAULT_INVOKE_TIMEOUT = 60 * 2 * 1000; // 2 minutes const size_t P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT = 5000; // 5 seconds const size_t P2P_CHECKPOINT_LIST_RE_REQUEST = 300; // 5 minutes - const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS = 20; // Minimum uptime for checkpoint verification (mainnet: 20 blocks × 2 min = 40 min, should be enought to cover 10000 blocks) + const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS = 2; // Minimum uptime for checkpoint verification (mainnet: 2 blocks × 2 min = 4 min) const uint32_t P2P_CHECKPOINT_PEER_MIN_UPTIME_BLOCKS_TESTNET = 2; // Minimum uptime for checkpoint verification (testnet: 2 blocks × 2 min = 4 min relaxed for testing) // Checkpoint consensus configuration (Mainnet) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 440b23821..53bb5dd0c 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -3413,22 +3413,41 @@ namespace cn // Only create if we haven't already created this chunk if (current_height == chunk_end_height && current_height > current_covered_height) { - // Check if chunk already exists (might have been created during initial generation) + // Check if chunk already exists (might have been created during initial generation + // or as a prefetched placeholder). A prefetched chunk must be verified here. + bool chunk_is_prefetched = m_checkpoints.is_chunk_prefetched(chunk_index); crypto::Hash existing_chunk_hash = m_checkpoints.get_chunk_hash(chunk_index); - if (existing_chunk_hash == NULL_HASH) + + if (existing_chunk_hash == NULL_HASH || chunk_is_prefetched) { logger(INFO, BRIGHT_GREEN) << "Reached chunk boundary at height " << current_height << " (chunk " << chunk_index << ", blocks " << chunk_start_height - << " to " << chunk_end_height << "). Computing chunk hash..."; + << " to " << chunk_end_height << "). " + << (chunk_is_prefetched ? "Verifying prefetched hash..." : "Computing chunk hash..."); auto getBlockIdsFunc = [this](uint32_t startHeight, uint32_t maxCount) -> std::vector { return m_blockIndex.getBlockIds(startHeight, maxCount); }; - - if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height)) + + bool prefetch_mismatch = false; + if (!m_checkpoints.add_chunk_from_block_ids(getBlockIdsFunc, chunk_start_height, &prefetch_mismatch)) { - logger(WARNING, BRIGHT_YELLOW) << "Failed to compute chunk " << chunk_index - << " hash at height " << current_height; + if (prefetch_mismatch) + { + // Local chain diverges from the prefetched network consensus hash. + // The chunk has been re-queued as unverified; validate_unverified_chunks() + // will run M-of-K consensus and trigger a rollback if the network confirms + // the divergence. + logger(ERROR, BRIGHT_RED) + << "Chunk " << chunk_index << " PREFETCH MISMATCH at height " << current_height + << "! Local hash differs from prefetched network consensus. " + << "Consensus validation will trigger rollback if network confirms divergence."; + } + else + { + logger(WARNING, BRIGHT_YELLOW) << "Failed to compute chunk " << chunk_index + << " hash at height " << current_height; + } } } else diff --git a/src/CryptoNoteCore/CheckpointList.h b/src/CryptoNoteCore/CheckpointList.h index 4790ac300..2fa33b3ec 100644 --- a/src/CryptoNoteCore/CheckpointList.h +++ b/src/CryptoNoteCore/CheckpointList.h @@ -56,9 +56,32 @@ namespace cn // This computes the hash for the latest chunk and stores it in memory only // NOTE: Chunk is NOT saved to checkpoint.dat until peer consensus is reached // After peer consensus, call add_verified_chunk_to_file() to save it + // + // out_prefetch_mismatch (optional): set to true when a prefetched consensus hash + // existed for this chunk but the locally computed hash disagreed. When this flag + // is set the chunk is stored with the local hash and queued for consensus validation + // (which will trigger a rollback if the network disagrees with the local chain). bool add_chunk_from_block_ids( std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc, - uint32_t chunk_start_height); + uint32_t chunk_start_height, + bool* out_prefetch_mismatch = nullptr); + + // --- Forward-prefetch API --- + // Add a chunk hash obtained from P2P consensus BEFORE the local chain has reached + // that chunk boundary. This extends the checkpoint zone so IBD can continue at + // fast speed. The hash is verified locally when add_chunk_from_block_ids() fires + // at the corresponding chunk boundary. + // Returns false if chunk already has a locally-computed (non-prefetched) hash. + bool add_prefetched_chunk(uint32_t chunk_index, const crypto::Hash& consensus_hash); + + // Returns true if the chunk at chunk_index was prefetched from the network and has + // not yet been confirmed by local computation. + bool is_chunk_prefetched(uint32_t chunk_index) const; + + // Returns the next chunk index that should be fetched from the network, given the + // estimated network tip height. Returns UINT32_MAX when nothing to prefetch + // (all chunks present, or the network tip is not far enough ahead). + uint32_t get_next_prefetchable_chunk_index(uint32_t network_height) const; // Add a verified chunk to checkpoint.dat (marks it as confirmed) // This is called after peer consensus is reached (M out of K peers agree) @@ -279,6 +302,7 @@ namespace cn const std::lock_guard lock(m_chunks_lock); m_chunks.clear(); m_confirmed_chunks.clear(); + m_prefetched_chunk_indices.clear(); } } @@ -526,6 +550,11 @@ namespace cn // and for answering checkpoint requests // Key: chunk_index, Value: true if confirmed via peer consensus std::unordered_set m_confirmed_chunks; + + // Prefetch tracking: chunk indices whose hash was obtained from P2P consensus + // BEFORE the local chain reached that boundary. When add_chunk_from_block_ids() + // fires for such an index it verifies the local hash against the stored value. + std::unordered_set m_prefetched_chunk_indices; // Legacy storage for backward compatibility (will be removed after migration) mutable std::mutex m_points_lock; diff --git a/src/CryptoNoteCore/CheckpointsList.cpp b/src/CryptoNoteCore/CheckpointsList.cpp index 2e5cf200e..64ea83a2c 100644 --- a/src/CryptoNoteCore/CheckpointsList.cpp +++ b/src/CryptoNoteCore/CheckpointsList.cpp @@ -555,26 +555,25 @@ namespace cn { */ bool CheckpointList::add_chunk_from_block_ids( std::function(uint32_t startHeight, uint32_t maxCount)> getBlockIdsFunc, - uint32_t chunk_start_height) + uint32_t chunk_start_height, + bool* out_prefetch_mismatch) { + if (out_prefetch_mismatch) *out_prefetch_mismatch = false; + + bool do_auto_confirm = false; // set inside lock when prefetch matches local hash + uint32_t auto_confirm_chunk_index = 0; + { const std::lock_guard lock(m_chunks_lock); - // SIMPLIFIED: Calculate chunk index and boundaries // All chunks are uniform: chunk_size blocks each (block 0/genesis excluded) // chunk[0]: blocks 1 to chunk_size // chunk[1]: blocks (chunk_size + 1) to (2 * chunk_size) // chunk[n]: blocks (n * chunk_size + 1) to ((n + 1) * chunk_size) - // - // Formula: chunk_index = (chunk_start_height - 1) / chunk_size - // Example: chunk_start_height = 1 → chunk_index = 0 ✓ - // chunk_start_height = 10001 → chunk_index = 10000 / 10000 = 1 ✓ - // chunk_start_height = 20001 → chunk_index = 20000 / 10000 = 2 ✓ uint32_t chunk_index = (chunk_start_height - 1) / m_chunk_size; uint32_t chunk_end_height = (chunk_index + 1) * m_chunk_size; uint32_t blocks_in_chunk = m_chunk_size; - // Get all block IDs for this chunk from blockchain.dat std::vector chunk_block_ids = getBlockIdsFunc(chunk_start_height, blocks_in_chunk); if (chunk_block_ids.size() != blocks_in_chunk) @@ -592,12 +591,10 @@ namespace cn { chunk_end_height, chunk_block_ids, getBlockIdsFunc, - false); // track_checkpoint_heights = false (only need counters) + false); if (!checkpoint_result.success) - { return false; - } uint32_t total_checkpoints_applied = checkpoint_result.checkpoints_from_config + checkpoint_result.checkpoints_from_dns; if (total_checkpoints_applied > 0) @@ -609,29 +606,70 @@ namespace cn { << checkpoint_result.checkpoints_from_dns << " from DNS, rest from blockchain.dat"; } - // Compute hash of this chunk's block IDs - crypto::Hash chunk_hash = crypto::cn_fast_hash( + crypto::Hash local_hash = crypto::cn_fast_hash( chunk_block_ids.data(), chunk_block_ids.size() * sizeof(crypto::Hash) ); - - // Ensure we have enough space in m_chunks - if (chunk_index >= m_chunks.size()) + + // --- Prefetch verification --- + // If this chunk was previously prefetched from the network, verify the local + // computation matches. A match lets us confirm immediately; a mismatch means + // our local chain diverged from the network — queue for consensus validation + // which will trigger a rollback if the network disagrees. + bool was_prefetched = (m_prefetched_chunk_indices.find(chunk_index) != m_prefetched_chunk_indices.end()); + if (was_prefetched) { - m_chunks.resize(chunk_index + 1); + crypto::Hash prefetched_hash = (chunk_index < m_chunks.size()) ? m_chunks[chunk_index] : NULL_HASH; + + if (local_hash == prefetched_hash) + { + // Match: local chain agrees with network consensus. Schedule confirmation. + m_prefetched_chunk_indices.erase(chunk_index); + m_chunks[chunk_index] = local_hash; // same value, but now locally owned + do_auto_confirm = true; + auto_confirm_chunk_index = chunk_index; + logger(INFO, logging::BRIGHT_GREEN) + << "Chunk " << chunk_index + << " locally verified — matches prefetched consensus hash. Confirming to checkpoint.dat."; + } + else + { + // Mismatch: local chain diverges from what the network agreed on. + // Replace the prefetched hash with the local one and remove the prefetch + // mark so validate_unverified_chunks() picks it up for consensus. + // If the network still disagrees, divergent_consensus will trigger a rollback. + logger(ERROR, logging::BRIGHT_RED) + << "Chunk " << chunk_index << " PREFETCH MISMATCH: " + << "local=" << local_hash << " prefetched=" << prefetched_hash + << ". Queueing for consensus validation (rollback likely)."; + m_prefetched_chunk_indices.erase(chunk_index); + m_chunks[chunk_index] = local_hash; + if (out_prefetch_mismatch) *out_prefetch_mismatch = true; + // Return true — chunk is stored; validate_unverified_chunks() handles rollback. + } + } + else + { + // Normal path: no prefetch record for this chunk. + if (chunk_index >= m_chunks.size()) + m_chunks.resize(chunk_index + 1); + m_chunks[chunk_index] = local_hash; + logger(INFO) << "Computed chunk " << chunk_index + << " hash (blocks " << chunk_start_height << "-" << chunk_end_height << ")" + << " - stored in memory (pending peer consensus)"; + } + } // m_chunks_lock released + + // Auto-confirm outside the lock (add_verified_chunk_to_file takes its own lock) + if (do_auto_confirm) + { + if (!add_verified_chunk_to_file(auto_confirm_chunk_index)) + { + logger(ERROR) << "Failed to save auto-confirmed prefetch chunk " + << auto_confirm_chunk_index << " to checkpoint.dat"; } - - m_chunks[chunk_index] = chunk_hash; - - logger(INFO) << "Computed chunk " << chunk_index - << " hash (blocks " << chunk_start_height << "-" << chunk_end_height << ")" - << " - stored in memory"; } - - // NOTE: We do NOT save to checkpoint.dat here. - // For chunks within the hardcoded-checkpoint range the caller saves immediately via - // add_verified_chunk_to_file(). For chunks beyond that range the caller waits for - // peer consensus before calling add_verified_chunk_to_file(). + return true; } @@ -763,6 +801,88 @@ namespace cn { return success; } + // --------------------------------------------------------------------------- + // Forward-prefetch methods + // --------------------------------------------------------------------------- + + /** + * Store a chunk hash that was obtained from P2P consensus before the local chain + * has reached that chunk boundary. Extends is_in_checkpoint_zone() so IBD can + * continue at fast speed. The hash is verified locally when + * add_chunk_from_block_ids() fires at the corresponding chunk boundary. + */ + bool CheckpointList::add_prefetched_chunk(uint32_t chunk_index, const crypto::Hash& consensus_hash) + { + if (consensus_hash == NULL_HASH) + { + logger(WARNING) << "Prefetch rejected for chunk " << chunk_index << ": consensus_hash is NULL"; + return false; + } + + const std::lock_guard lock(m_chunks_lock); + + // Don't overwrite a chunk that has already been locally computed (not prefetched). + if (chunk_index < m_chunks.size() && m_chunks[chunk_index] != NULL_HASH + && m_prefetched_chunk_indices.find(chunk_index) == m_prefetched_chunk_indices.end()) + { + logger(DEBUGGING) << "Prefetch skipped for chunk " << chunk_index + << ": already has a locally-computed hash"; + return false; + } + + if (chunk_index >= m_chunks.size()) + m_chunks.resize(chunk_index + 1, NULL_HASH); + + m_chunks[chunk_index] = consensus_hash; + m_prefetched_chunk_indices.insert(chunk_index); + + uint32_t covered_up_to = (chunk_index + 1) * m_chunk_size; + logger(INFO, logging::BRIGHT_CYAN) + << "Prefetched chunk " << chunk_index + << " from P2P consensus (covers up to block " << covered_up_to + << ", hash: " << consensus_hash << ")"; + return true; + } + + /** + * Returns true when the chunk at chunk_index holds a hash obtained from P2P + * consensus rather than local block-id computation. + */ + bool CheckpointList::is_chunk_prefetched(uint32_t chunk_index) const + { + const std::lock_guard lock(m_chunks_lock); + return m_prefetched_chunk_indices.find(chunk_index) != m_prefetched_chunk_indices.end(); + } + + /** + * Returns the index of the next chunk that should be fetched from the network. + * A chunk can be prefetched only when the network tip is high enough for that + * chunk to be complete (network_height >= (chunk_index + 1) * chunk_size). + * Returns UINT32_MAX when nothing to prefetch. + */ + uint32_t CheckpointList::get_next_prefetchable_chunk_index(uint32_t network_height) const + { + const std::lock_guard lock(m_chunks_lock); + + // Find the first index with a missing/null entry. + uint32_t next = static_cast(m_chunks.size()); // default: one past the end + for (uint32_t i = 0; i < static_cast(m_chunks.size()); i++) + { + if (m_chunks[i] == NULL_HASH) { next = i; break; } + } + + // The chunk is complete on the network only when network_height covers all its blocks: + // chunk 'next' covers blocks (next * chunk_size + 1) to (next + 1) * chunk_size + // → complete when network_height >= (next + 1) * chunk_size + uint64_t required = static_cast(next + 1) * m_chunk_size; + if (static_cast(network_height) < required) + return UINT32_MAX; + + return next; + } + + // --------------------------------------------------------------------------- + /** * Get the hash of a specific chunk * diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp index b13754480..cff430a55 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandler.cpp @@ -683,14 +683,19 @@ int CryptoNoteProtocolHandler::processObjects(CryptoNoteConnectionContext& conte bool CryptoNoteProtocolHandler::on_idle() { - // Periodically validate unverified chunks in chronological order - // Only for version 2+ nodes (chunked checkpoint system) + // Periodically validate unverified chunks and prefetch future chunk hashes. + // Only for version 2+ nodes (chunked checkpoint system). if (cn::P2P_CURRENT_VERSION >= cn::P2P_CHECKPOINT_LIST_VERSION) { - // Check pending validations for consensus (asynchronous approach) m_chunkValidationManager->check_pending_chunk_validations(); - // Start new validations if needed m_chunkValidationManager->validate_unverified_chunks(); + + // Forward prefetch: fetch chunk hashes the network already has but we don't yet. + // This extends the checkpoint zone so IBD stays at fast speed. + // Observed height is local_tip + 1 per peer convention; subtract 1 for actual tip. + uint32_t obs = getObservedHeight(); + uint32_t net_tip = (obs > 0) ? (obs - 1) : 0; + m_chunkValidationManager->prefetch_missing_chunks(net_tip); } return m_core.on_idle(); diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index 914108b48..ff2c9505e 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -54,6 +54,7 @@ ChunkValidationManager::ChunkValidationManager(ICore& core, , m_stop(stop) , m_current_validating_chunk_index(UINT32_MAX) , m_last_chunk_validation_attempt(0) + , m_last_prefetch_attempt(0) { } @@ -439,6 +440,7 @@ bool ChunkValidationManager::validate_unverified_chunks() pending.peer_network_16 = sent_peer_networks; pending.local_hash = local_chunk_hash; pending.is_first_attempt = true; + pending.is_prefetch = false; m_pending_validations[chunk_index] = pending; } @@ -537,6 +539,34 @@ void ChunkValidationManager::check_pending_chunk_validations() << vote_result.local_diverse_networks << "), " << "NULL_HASH responses: " << vote_result.null_hash_responses; + // --- Prefetch success path --- + // For prefetch rounds local_hash == NULL_HASH, so local_consensus is always false. + // Success is when M peers agree on a real (non-null) hash from enough networks, + // which evaluate_consensus_votes() reports as divergent_consensus. + if (pending.is_prefetch && vote_result.divergent_consensus + && vote_result.consensus_hash != NULL_HASH) + { + logger(INFO, BRIGHT_CYAN) + << "[Chunk Prefetch] Chunk " << chunk_index + << " consensus reached: " << vote_result.consensus_hash_votes + << " peers from " << vote_result.consensus_hash_diverse_networks + << " networks agree on hash " << vote_result.consensus_hash; + + crypto::Hash prefetch_hash = vote_result.consensus_hash; + std::vector peers_to_cleanup = pending.requested_peers; + it = m_pending_validations.erase(it); + cleanup_chunk_responses(chunk_index, peers_to_cleanup); + + // Store the prefetched hash (extends checkpoint zone immediately) + if (!m_core.getCheckpointList().add_prefetched_chunk(chunk_index, prefetch_hash)) + { + logger(WARNING) << "[Chunk Prefetch] add_prefetched_chunk(" << chunk_index << ") failed " + << "(chunk may already be locally computed — this is fine)"; + } + continue; + } + + // --- Normal validation success path --- // Check if we have M agreements from enough networks (consensus reached) if (vote_result.local_consensus) { @@ -562,11 +592,12 @@ void ChunkValidationManager::check_pending_chunk_validations() if (vote_result.responses_received == 0 || (vote_result.responses_received == vote_result.null_hash_responses && !vote_result.divergent_consensus)) { - logger(INFO) << "Chunk " << chunk_index - << " validation: No peers have this chunk in memory yet (all returned NULL_HASH or no response). " + const char* mode = pending.is_prefetch ? "prefetch" : "validation"; + logger(INFO) << "Chunk " << chunk_index + << " " << mode << ": No peers have this chunk in memory yet (all returned NULL_HASH or no response). " << "This is normal if: (1) peers are using version 1 (don't support chunk checkpoints), " << "or (2) peers haven't created this chunk yet. " - << "Will retry validation once peers create this chunk."; + << "Will retry once peers have this chunk."; std::vector peers_to_cleanup = pending.requested_peers; it = m_pending_validations.erase(it); @@ -615,6 +646,7 @@ void ChunkValidationManager::check_pending_chunk_validations() pending.requested_peers = sent_peers; pending.peer_network_16 = sent_peer_networks; pending.is_first_attempt = false; + // is_prefetch is preserved from the first attempt (already set) logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index << " to " << sent_peers.size() << " peer(s). " @@ -730,5 +762,145 @@ void ChunkValidationManager::check_pending_chunk_validations() } } +// --------------------------------------------------------------------------- +// Forward prefetch: fetch chunk hashes ahead of local chain tip +// --------------------------------------------------------------------------- + +bool ChunkValidationManager::prefetch_missing_chunks(uint32_t network_height) +{ + // Derive the actual network tip: peers advertise local_tip + 1, so subtract 1. + uint32_t net_tip = (network_height > 0) ? (network_height - 1) : 0; + + // Ask CheckpointList for the next chunk index we should prefetch. + uint32_t chunk_index = m_core.getCheckpointList().get_next_prefetchable_chunk_index(net_tip); + if (chunk_index == UINT32_MAX) + return false; // Nothing to prefetch + + // Don't duplicate an in-flight request. + if (is_chunk_being_validated(chunk_index)) + return false; + + // Already prefetched? + if (m_core.getCheckpointList().is_chunk_prefetched(chunk_index)) + return false; + + // Rate-limit: attempt at most once every 60 seconds. + uint64_t time_now = time(nullptr); + { + std::lock_guard lock(m_chunk_validation_mutex); + if (time_now - m_last_prefetch_attempt < 60) + return false; + m_last_prefetch_attempt = time_now; + } + + if (m_peersCount.load() == 0) + return false; + + // Collect eligible peers (same rules as validate_unverified_chunks). + uint32_t current_height; + crypto::Hash top_id; + m_core.get_blockchain_top(current_height, top_id); + uint32_t block_time = m_currency.difficultyTarget(); + uint32_t min_uptime_blocks = m_core.getCheckpointList().get_min_peer_uptime_blocks(); + + std::vector eligible_peers; + std::map peer_network_16; + + m_p2p->for_each_connection([&](CryptoNoteConnectionContext& ctx, uint64_t peer_id) { + if (ctx.version < cn::P2P_CHECKPOINT_LIST_VERSION) return; + if (ctx.m_state != CryptoNoteConnectionContext::state_normal + && ctx.m_state != CryptoNoteConnectionContext::state_idle + && ctx.m_state != CryptoNoteConnectionContext::state_synchronizing) return; + time_t duration = time_now - ctx.m_started; + if (duration < 0) return; + uint32_t uptime_blocks = static_cast(duration / block_time); + if (uptime_blocks < min_uptime_blocks) return; + eligible_peers.push_back(peer_id); + peer_network_16[peer_id] = CheckpointList::get_network_16(ctx.m_remote_ip); + }); + + if (eligible_peers.empty()) + { + logger(DEBUGGING) << "[Chunk Prefetch] No eligible peers for chunk " << chunk_index; + return false; + } + + CheckpointList::ConsensusRequirements req = + m_core.getCheckpointList().calculate_consensus_requirements(eligible_peers.size()); + + if (eligible_peers.size() < req.min_peers) + { + logger(DEBUGGING) << "[Chunk Prefetch] Not enough eligible peers: have " + << eligible_peers.size() << ", need K=" << req.min_peers; + return false; + } + + auto getPeerNetwork16 = [&peer_network_16](uint64_t pid) -> uint32_t { + auto it = peer_network_16.find(pid); + return (it != peer_network_16.end()) ? it->second : 0; + }; + + CheckpointList::PeerSamplingResult sampling = + CheckpointList::sample_peers_with_diversity(eligible_peers, getPeerNetwork16, req.min_peers); + + if (sampling.sampled_peers.empty()) + { + logger(WARNING) << "[Chunk Prefetch] Could not sample peers for chunk " << chunk_index; + return false; + } + + uint32_t distinct_networks = static_cast(sampling.network_votes.size()); + if (distinct_networks < req.min_diverse_networks) + { + logger(DEBUGGING) << "[Chunk Prefetch] Insufficient network diversity for chunk " << chunk_index + << ": have " << distinct_networks + << " distinct /16, need " << req.min_diverse_networks; + return false; + } + + // Send async requests. + std::vector sent_peers; + std::map sent_networks; + for (uint64_t pid : sampling.sampled_peers) + { + if (send_chunk_hash_request_async(pid, chunk_index)) + { + sent_peers.push_back(pid); + sent_networks[pid] = getPeerNetwork16(pid); + } + } + + if (sent_peers.size() < req.min_peers) + { + logger(WARNING) << "[Chunk Prefetch] Only sent " << sent_peers.size() + << " request(s) for chunk " << chunk_index + << ", need K=" << req.min_peers; + cleanup_chunk_responses(chunk_index, sent_peers); + return false; + } + + { + std::lock_guard lock(m_pending_validations_mutex); + PendingChunkValidation pending; + pending.chunk_index = chunk_index; + pending.request_timestamp = time_now; + pending.attempt_start_time = time_now; + pending.attempt_number = 1; + pending.requested_peers = sent_peers; + pending.peer_network_16 = sent_networks; + pending.local_hash = NULL_HASH; // no local hash yet + pending.is_first_attempt = true; + pending.is_prefetch = true; + m_pending_validations[chunk_index] = pending; + } + + logger(INFO, BRIGHT_CYAN) + << "[Chunk Prefetch] Requested chunk " << chunk_index + << " from " << sent_peers.size() << " peers across " << distinct_networks + << " networks. Will check consensus in 3 minutes."; + + return true; +} + } // namespace cn diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h index 7fbdd143a..b1613dc3c 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.h @@ -53,6 +53,14 @@ namespace cn // Checks if we have M identical hashes within the time window (3 minutes) // If consensus reached, processes it; if timeout, schedules retry void check_pending_chunk_validations(); + + // Prefetch chunk hashes ahead of local chain to keep the checkpoint zone extended. + // Queries network peers for the next missing chunk hash; if M-of-K peers agree, + // stores it as a prefetched chunk so IBD can continue at fast speed. + // Called from on_idle() alongside validate_unverified_chunks(). + // @param network_height Estimated network tip (observed height - 1) + // @return true if a prefetch round was started + bool prefetch_missing_chunks(uint32_t network_height); // Send chunk hash request asynchronously (non-blocking) // Returns true if request was sent successfully @@ -83,6 +91,10 @@ namespace cn std::map peer_network_16; // peer_id -> /16 network crypto::Hash local_hash; bool is_first_attempt; + // is_prefetch == true: we asked the network for a chunk we don't have locally yet. + // local_hash is NULL_HASH; success means M peers agreed on a non-null hash that + // we store as a prefetched chunk to extend the checkpoint zone. + bool is_prefetch; }; void cleanup_chunk_responses(uint32_t chunk_index, const std::vector& peer_ids); @@ -107,6 +119,7 @@ namespace cn mutable std::mutex m_chunk_validation_mutex; uint32_t m_current_validating_chunk_index; // Currently validating chunk (or UINT32_MAX if none) uint64_t m_last_chunk_validation_attempt; // Last time we attempted validation (to avoid spamming) + uint64_t m_last_prefetch_attempt; // Last time prefetch_missing_chunks() ran }; } // namespace cn From 2f04fe0f39e8258b2bfe28008721b84fa99d0b02 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 5 May 2026 15:58:44 -0400 Subject: [PATCH 54/56] simplify logging --- src/CryptoNoteCore/Blockchain.cpp | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/CryptoNoteCore/Blockchain.cpp b/src/CryptoNoteCore/Blockchain.cpp index 53bb5dd0c..61e370af1 100644 --- a/src/CryptoNoteCore/Blockchain.cpp +++ b/src/CryptoNoteCore/Blockchain.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -1012,17 +1013,10 @@ namespace cn std::vector unverified = m_checkpoints.get_unverified_chunks(); if (!unverified.empty()) { - logger(INFO) << "Found " << unverified.size() << " unverified chunk(s) requiring P2P validation: chunks " - << unverified[0]; - for (size_t i = 1; i < unverified.size() && i < 5; i++) - { - logger(INFO) << ", " << unverified[i]; - } - if (unverified.size() > 5) - { - logger(INFO) << ", ... (and " << (unverified.size() - 5) << " more)"; - } - logger(INFO) << ". These will be validated via peer consensus before being saved to checkpoint.dat."; + if (unverified.size() == 1) + logger(INFO) << "Found 1 unverified chunk: " << unverified[0]; + else + logger(INFO) << "Found " << unverified.size() << " unverified chunks from " << unverified[0]; } } } From e5f7227a97170d3c1026a7634ac1957f5b267ab8 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 5 May 2026 16:15:37 -0400 Subject: [PATCH 55/56] add permissions to workflows --- .github/workflows/check.yml | 15 ++++++++++++--- .github/workflows/macOS.yml | 5 ++--- .github/workflows/ubuntu22.yml | 5 ++--- .github/workflows/ubuntu24.yml | 5 ++--- .github/workflows/windows.yml | 5 ++--- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index d533d93dc..9590b69ed 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -8,13 +8,12 @@ on: - "*" # We don't want this to run on release pull_request: -permissions: - contents: write - jobs: build-windows: name: Windows runs-on: windows-2022 + permissions: + contents: write env: BOOST_ROOT: C:/local/boost_1_83_0 steps: @@ -102,6 +101,8 @@ jobs: build-mingw: name: MinGW runs-on: windows-2022 + permissions: + contents: write defaults: run: shell: msys2 {0} @@ -186,6 +187,8 @@ jobs: build-ubuntu22: name: Ubuntu 22.04 runs-on: ubuntu-22.04 + permissions: + contents: write steps: - uses: actions/checkout@v5 @@ -256,6 +259,8 @@ jobs: build-ubuntu24: name: Ubuntu 24.04 runs-on: ubuntu-24.04 + permissions: + contents: write steps: - uses: actions/checkout@v5 @@ -326,6 +331,8 @@ jobs: build-ubuntu22-clang: name: Ubuntu 22.04 clang runs-on: ubuntu-22.04 + permissions: + contents: write steps: - uses: actions/checkout@v5 @@ -396,6 +403,8 @@ jobs: build-macos: name: macOS runs-on: macos-15-intel + permissions: + contents: write steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 06157842f..e74487bcd 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -5,13 +5,12 @@ on: tags: - "*" -permissions: - contents: write - jobs: build-macos: name: macOS runs-on: macos-15-intel + permissions: + contents: write steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ubuntu22.yml b/.github/workflows/ubuntu22.yml index 19e8561ad..9bcc377c6 100644 --- a/.github/workflows/ubuntu22.yml +++ b/.github/workflows/ubuntu22.yml @@ -5,13 +5,12 @@ on: tags: - "*" -permissions: - contents: write - jobs: build-ubuntu22: name: Ubuntu 22.04 runs-on: ubuntu-22.04 + permissions: + contents: write steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/ubuntu24.yml b/.github/workflows/ubuntu24.yml index d163b056c..9849e0673 100644 --- a/.github/workflows/ubuntu24.yml +++ b/.github/workflows/ubuntu24.yml @@ -5,13 +5,12 @@ on: tags: - "*" -permissions: - contents: write - jobs: build-ubuntu24: name: Ubuntu 24.04 runs-on: ubuntu-24.04 + permissions: + contents: write steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index aa6847dd9..8dbfe7e7e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -5,13 +5,12 @@ on: tags: - "*" -permissions: - contents: write - jobs: build-windows: name: Windows runs-on: windows-2022 + permissions: + contents: write env: BOOST_ROOT: C:/local/boost_1_83_0 steps: From 5926f32f48868fdd752209e311e84cdb19ef8261 Mon Sep 17 00:00:00 2001 From: acktarius Date: Tue, 5 May 2026 16:28:04 -0400 Subject: [PATCH 56/56] adjust consensus wait period to 60 seconds, retry in 30s --- .../CryptoNoteProtocolHandlerChunk.cpp | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp index ff2c9505e..b0f6d734f 100644 --- a/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp +++ b/src/CryptoNoteProtocol/CryptoNoteProtocolHandlerChunk.cpp @@ -140,7 +140,7 @@ bool ChunkValidationManager::is_chunk_being_validated(uint32_t chunk_index) cons { // Check if chunk is in pending validations (actual validation state) // m_current_validating_chunk_index is cleared immediately after sending requests, - // but validation is still pending for 3 minutes, so we need to check m_pending_validations + // but validation is still pending for up to 60 seconds, so we need to check m_pending_validations std::lock_guard lock(m_pending_validations_mutex); return (m_pending_validations.find(chunk_index) != m_pending_validations.end()); } @@ -324,6 +324,20 @@ bool ChunkValidationManager::validate_unverified_chunks() m_current_validating_chunk_index = chunk_index; } + // Prefetched chunks are auto-confirmed when the local chain reaches the chunk + // boundary (add_chunk_from_block_ids verifies them). Running a separate peer + // consensus round on them is redundant and misleading — skip. + if (m_core.getCheckpointList().is_chunk_prefetched(chunk_index)) + { + logger(DEBUGGING) << "[Chunk Validation] Chunk " << chunk_index + << " is prefetched — skipping (auto-confirmed by local computation)"; + { + std::lock_guard lock(m_chunk_validation_mutex); + m_current_validating_chunk_index = UINT32_MAX; + } + continue; + } + // Get local chunk hash crypto::Hash local_chunk_hash = m_core.getCheckpointList().get_chunk_hash(chunk_index); if (local_chunk_hash == NULL_HASH) @@ -444,7 +458,7 @@ bool ChunkValidationManager::validate_unverified_chunks() m_pending_validations[chunk_index] = pending; } - logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << sent_peers.size() << " peers. Checking consensus in 3 minutes."; + logger(INFO) << "[Chunk Validation] Sent chunk " << chunk_index << " hash requests to " << sent_peers.size() << " peers. Checking consensus in 60 seconds."; // Clear validation state (validation is now async - will be checked in check_pending_chunk_validations) { @@ -463,8 +477,8 @@ bool ChunkValidationManager::validate_unverified_chunks() void ChunkValidationManager::check_pending_chunk_validations() { uint64_t time_now = time(nullptr); - const uint64_t CONSENSUS_WAIT_SECONDS = 180; // 3 minutes (increased from 2 to handle network latency and peer processing) - const uint64_t RETRY_DELAY_SECONDS = 60; // 1 minute delay before retry + const uint64_t CONSENSUS_WAIT_SECONDS = 60; // 60 seconds + const uint64_t RETRY_DELAY_SECONDS = 30; // 30 seconds delay before retry struct PendingAction { @@ -502,7 +516,9 @@ void ChunkValidationManager::check_pending_chunk_validations() } // 3 minutes have passed - check for consensus - logger(INFO) << "[Chunk Validation] Checking consensus for chunk " << chunk_index << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; + if (!pending.is_prefetch) + logger(INFO) << "[Chunk Validation] Checking consensus for chunk " << chunk_index + << " (elapsed: " << elapsed << " seconds, attempt " << pending.attempt_number << ")"; // Collect fresh responses from requested peers std::vector votes; @@ -532,12 +548,13 @@ void ChunkValidationManager::check_pending_chunk_validations() pending.request_timestamp, req); - logger(INFO) << "[Chunk Validation] Chunk " << chunk_index << " consensus check: received " << vote_result.responses_received - << " fresh response(s) from " << pending.requested_peers.size() << " requested peer(s). " - << "Agreements: " << vote_result.agreements << " (need M=" << req.min_agreements - << " from n=" << req.min_diverse_networks << " networks, got n=" - << vote_result.local_diverse_networks << "), " - << "NULL_HASH responses: " << vote_result.null_hash_responses; + if (!pending.is_prefetch) + logger(INFO) << "[Chunk Validation] Chunk " << chunk_index << " consensus check: received " << vote_result.responses_received + << " fresh response(s) from " << pending.requested_peers.size() << " requested peer(s). " + << "Agreements: " << vote_result.agreements << " (need M=" << req.min_agreements + << " from n=" << req.min_diverse_networks << " networks, got n=" + << vote_result.local_diverse_networks << "), " + << "NULL_HASH responses: " << vote_result.null_hash_responses; // --- Prefetch success path --- // For prefetch rounds local_hash == NULL_HASH, so local_consensus is always false. @@ -650,7 +667,7 @@ void ChunkValidationManager::check_pending_chunk_validations() logger(INFO) << "Sent second attempt async requests for chunk " << chunk_index << " to " << sent_peers.size() << " peer(s). " - << "Will check for consensus after 3 minutes."; + << "Will check for consensus after 60 seconds."; } else { @@ -897,7 +914,7 @@ bool ChunkValidationManager::prefetch_missing_chunks(uint32_t network_height) logger(INFO, BRIGHT_CYAN) << "[Chunk Prefetch] Requested chunk " << chunk_index << " from " << sent_peers.size() << " peers across " << distinct_networks - << " networks. Will check consensus in 3 minutes."; + << " networks. Will check consensus in 60 seconds."; return true; }