diff --git a/doc/rpc-maturity.md b/doc/rpc-maturity.md index d71d38959c5..7fd873dc3e3 100644 --- a/doc/rpc-maturity.md +++ b/doc/rpc-maturity.md @@ -28,6 +28,7 @@ Core RPC. Maturity is expressed over 3 stages: | decoderawtransaction | STABLE | | | decodescript | STABLE | | | disconnectnode | STABLE | | +| dumptxoutset | UNSTABLE | New in 1.14.99 | | dumpprivkey | STABLE | | | dumpwallet | UNSTABLE | Breaking change since 1.14.6 | | encryptwallet | STABLE | | @@ -97,6 +98,7 @@ Core RPC. Maturity is expressed over 3 stages: | listtransactions | STABLE | | | listunspent | STABLE | | | lockunspent | STABLE | | +| loadtxoutset | UNSTABLE | New in 1.14.99 | | move | DEPRECATED | Deprecated since 1.14.0 | | ping | STABLE | | | preciousblock | STABLE | | diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index ad4beae7083..0df557b08a8 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -148,6 +148,8 @@ 'nodehandling.py', 'decodescript.py', 'blockchain.py', + 'dumptxoutset.py', + 'loadtxoutset.py', 'disablewallet.py', 'keypool.py', 'p2p-mempool.py', diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index 7c3b3ad5d49..18ddc8dbfab 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -108,9 +108,9 @@ def _test_gettxoutsetinfo(self): assert_equal(res['transactions'], 120) assert_equal(res['height'], 120) assert_equal(res['txouts'], 120) - assert_equal(res['bytes_serialized'], 8520), + assert_greater_than(res['disk_size'], 0) assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized']), 64) + assert_equal(len(res['hash_serialized_2']), 64) def _test_getblockheader(self): node = self.nodes[0] diff --git a/qa/rpc-tests/dumptxoutset.py b/qa/rpc-tests/dumptxoutset.py new file mode 100644 index 00000000000..1493a25b7bd --- /dev/null +++ b/qa/rpc-tests/dumptxoutset.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2024 The Dogecoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test the dumptxoutset RPC call. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_jsonrpc, + start_node, +) + + +class DumptxoutsetTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + def setup_network(self, split=False): + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, [])) + self.is_network_split = False + + def run_test(self): + """Test basic dumptxoutset functionality.""" + node = self.nodes[0] + + # Mine some blocks so there are UTXOs to dump + node.generate(110) + + # Determine the output path inside the node's datadir + datadir = os.path.join(self.options.tmpdir, "node0") + out_path = os.path.join(datadir, "utxos.dat") + temp_path = out_path + ".incomplete" + + # Call dumptxoutset + result = node.dumptxoutset(out_path) + + # Validate return fields + assert_equal(result['base_height'], 110) + assert_equal(len(result['base_hash']), 64) + assert_equal(result['path'], out_path) + assert_greater_than(result['coins_written'], 0) + # 110 coinbase transactions = 110 UTXOs (one per block) + assert_equal(result['coins_written'], 110) + # New fields added in session 5 + assert_equal(len(result['txoutset_hash']), 64) + assert_greater_than(result['nchaintx'], 0) + + # The final file should exist + assert os.path.exists(out_path), "snapshot file was not created" + + # The temporary (.incomplete) file should have been renamed away + assert not os.path.exists(temp_path), ".incomplete file still present" + + # File must be non-empty + assert_greater_than(os.path.getsize(out_path), 0) + + # Calling again with the same path should fail + assert_raises_jsonrpc( + -8, + "already exists", + node.dumptxoutset, + out_path, + ) + + # Passing a relative path returns an absolute path in the result + result2 = node.dumptxoutset("utxos2.dat") + assert os.path.isabs(result2['path']) + assert result2['path'].endswith("utxos2.dat") + assert os.path.exists(result2['path']) + + # Invalid output directories should fail cleanly + invalid_path = os.path.join(datadir, "missing", "utxos3.dat") + assert_raises_jsonrpc( + -8, + "Couldn't open file", + node.dumptxoutset, + invalid_path, + ) + + print("dumptxoutset tests passed") + + +if __name__ == '__main__': + DumptxoutsetTest().main() diff --git a/qa/rpc-tests/loadtxoutset.py b/qa/rpc-tests/loadtxoutset.py new file mode 100644 index 00000000000..5d8895dd3d2 --- /dev/null +++ b/qa/rpc-tests/loadtxoutset.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Dogecoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test the loadtxoutset RPC call. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_jsonrpc, + start_node, +) + + +class LoadtxoutsetTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + def setup_network(self, split=False): + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, [])) + self.is_network_split = False + + def run_test(self): + node = self.nodes[0] + + node.generate(110) + datadir = os.path.join(self.options.tmpdir, "node0") + snapshot_path = os.path.join(datadir, "utxos-load.dat") + + missing_snapshot = os.path.join(datadir, "does-not-exist.dat") + assert_raises_jsonrpc( + -8, + "Couldn't open file", + node.loadtxoutset, + missing_snapshot, + "00" * 32, + ) + + dump_result = node.dumptxoutset(snapshot_path) + snapshot_info = node.gettxoutsetinfo() + + assert_equal(dump_result["base_height"], 110) + assert_equal(snapshot_info["hash_serialized_2"], dump_result["txoutset_hash"]) + + bad_magic_path = snapshot_path + ".badmagic" + with open(snapshot_path, "rb") as snapshot_file: + snapshot_bytes = bytearray(snapshot_file.read()) + snapshot_bytes[0] ^= 0x01 + with open(bad_magic_path, "wb") as snapshot_file: + snapshot_file.write(snapshot_bytes) + assert_raises_jsonrpc( + -22, + "Unable to parse metadata: Invalid UTXO snapshot magic bytes", + node.loadtxoutset, + bad_magic_path, + dump_result["txoutset_hash"], + ) + + bad_network_path = snapshot_path + ".badnet" + with open(snapshot_path, "rb") as snapshot_file: + snapshot_bytes = bytearray(snapshot_file.read()) + snapshot_bytes[7:11] = b"\x00\x00\x00\x00" + with open(bad_network_path, "wb") as snapshot_file: + snapshot_file.write(snapshot_bytes) + assert_raises_jsonrpc( + -8, + "Snapshot network does not match this node", + node.loadtxoutset, + bad_network_path, + dump_result["txoutset_hash"], + ) + + node.generate(5) + assert_equal(node.getblockcount(), 115) + + assert_raises_jsonrpc( + -8, + "Snapshot content hash mismatch", + node.loadtxoutset, + snapshot_path, + "00" * 32, + ) + + load_result = node.loadtxoutset(snapshot_path, dump_result["txoutset_hash"]) + + assert_equal(load_result["coins_loaded"], dump_result["coins_written"]) + assert_equal(load_result["base_height"], dump_result["base_height"]) + assert_equal(load_result["base_hash"], dump_result["base_hash"]) + assert_equal(load_result["path"], snapshot_path) + + restored_info = node.gettxoutsetinfo() + assert_equal(node.getblockcount(), 110) + assert_equal(restored_info["bestblock"], snapshot_info["bestblock"]) + assert_equal(restored_info["transactions"], snapshot_info["transactions"]) + assert_equal(restored_info["txouts"], snapshot_info["txouts"]) + assert_equal(restored_info["hash_serialized_2"], snapshot_info["hash_serialized_2"]) + +if __name__ == '__main__': + LoadtxoutsetTest().main() diff --git a/src/Makefile.am b/src/Makefile.am index d2b7eedfb0a..78168c3f871 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -130,6 +130,7 @@ BITCOIN_CORE_H = \ netaddress.h \ netbase.h \ netmessagemaker.h \ + node/utxo_snapshot.h \ noui.h \ policy/fees.h \ policy/policy.h \ diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1e8e3d462fc..5aab3381fd2 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50 * CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21 * CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22 * CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 0a70e930314..b62b1621728 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -576,24 +576,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout must be positive"); + COutPoint out(txid, nOut); std::vector pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw std::runtime_error(err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]); + newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and private keys given, @@ -615,13 +617,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (!coins || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { fComplete = false; continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: diff --git a/src/coins.cpp b/src/coins.cpp index cda0ea5bd3b..27a24354785 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,174 +4,132 @@ #include "coins.h" +#include "consensus/consensus.h" #include "memusage.h" #include "random.h" #include -/** - * calculate number of bytes for the bitmask, and its number of non-zero bytes - * each bit in the bitmask represents the availability of one output, but the - * availabilities of the first two outputs are encoded separately - */ -void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const { - unsigned int nLastUsedByte = 0; - for (unsigned int b = 0; 2+b*8 < vout.size(); b++) { - bool fZero = true; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) { - if (!vout[2+b*8+i].IsNull()) { - fZero = false; - continue; - } - } - if (!fZero) { - nLastUsedByte = b + 1; - nNonzeroBytes++; - } - } - nBytes += nLastUsedByte; -} - -bool CCoins::Spend(uint32_t nPos) -{ - if (nPos >= vout.size() || vout[nPos].IsNull()) - return false; - vout[nPos].SetNull(); - Cleanup(); - return true; -} - -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const +{ + Coin coin; + return GetCoin(outpoint, coin); +} + CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } +size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} - -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} -CCoinsViewCache::~CCoinsViewCache() -{ - assert(!hasModifier); -} +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; - CCoins tmp; - if (!base->GetCoins(txid, tmp)) + Coin tmp; + if (!base->GetCoin(outpoint, tmp)) return cacheCoins.end(); - CCoinsMap::iterator ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); - if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our + CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; + if (ret->second.coin.IsSpent()) { + // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it != cacheCoins.end()) { - coins = it->second.coins; - return true; + coin = it->second.coin; + return !coin.IsSpent(); } return false; } -CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { - assert(!hasModifier); - std::pair ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - size_t cachedCoinUsage = 0; - if (ret.second) { - if (!base->GetCoins(txid, ret.first->second.coins)) { - // The parent view does not have this entry; mark it as fresh. - ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } else if (ret.first->second.coins.IsPruned()) { - // The parent view only has a pruned entry for this; mark it as fresh. - ret.first->second.flags = CCoinsCacheEntry::FRESH; +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { + assert(!coin.IsSpent()); + if (coin.out.scriptPubKey.IsUnspendable()) return; + CCoinsMap::iterator it; + bool inserted; + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); + bool fresh = false; + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } + if (!possible_overwrite) { + if (!it->second.coin.IsSpent()) { + throw std::logic_error("Adding new coin that replaces non-pruned entry"); } - } else { - cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } - // Assume that whenever ModifyCoins is called, the entry will be modified. - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, cachedCoinUsage); + it->second.coin = std::move(coin); + it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } -/* ModifyNewCoins allows for faster coin modification when creating the new - * outputs from a transaction. It assumes that BIP 30 (no duplicate txids) - * applies and has already been tested for (or the test is not required due to - * BIP 34, height in coinbase). If we can assume BIP 30 then we know that any - * non-coinbase transaction we are adding to the UTXO must not already exist in - * the utxo unless it is fully spent. Thus we can check only if it exists DIRTY - * at the current level of the cache, in which case it is not safe to mark it - * FRESH (b/c then its spentness still needs to flushed). If it's not dirty and - * doesn't exist or is pruned in the current cache, we know it either doesn't - * exist or is pruned in parent caches, which is the definition of FRESH. The - * exception to this is the two historical violations of BIP 30 in the chain, - * both of which were coinbases. We do not mark these fresh so we we can ensure - * that they will still be properly overwritten when spent. - */ -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { - assert(!hasModifier); - std::pair ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - if (!coinbase) { - // New coins must not already exist. - if (!ret.first->second.coins.IsPruned()) - throw std::logic_error("ModifyNewCoins should not find pre-existing coins on a non-coinbase unless they are pruned!"); - - if (!(ret.first->second.flags & CCoinsCacheEntry::DIRTY)) { - // If the coin is known to be pruned (have no unspent outputs) in - // the current view and the cache entry is not dirty, we know the - // coin also must be pruned in the parent view as well, so it is safe - // to mark this fresh. - ret.first->second.flags |= CCoinsCacheEntry::FRESH; - } +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { + bool fCoinbase = tx.IsCoinBase(); + const uint256& txid = tx.GetHash(); + for (size_t i = 0; i < tx.vout.size(); ++i) { + // Pass fCoinbase as the possible_overwrite flag to AddCoin, in order to correctly + // deal with the pre-BIP30 occurrances of duplicate coinbase transactions. + cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), fCoinbase); } - ret.first->second.coins.Clear(); - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, 0); } -const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) return false; + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + if (moveout) { + *moveout = std::move(it->second.coin); + } + if (it->second.flags & CCoinsCacheEntry::FRESH) { + cacheCoins.erase(it); + } else { + it->second.flags |= CCoinsCacheEntry::DIRTY; + it->second.coin.Clear(); + } + return true; +} + +static const Coin coinEmpty; + +const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { - return NULL; + return coinEmpty; } else { - return &it->second.coins; + return it->second.coin; } } -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } -bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { - CCoinsMap::const_iterator it = cacheCoins.find(txid); +bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -186,19 +144,18 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coins.IsPruned())) { + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache @@ -211,21 +168,21 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coins.IsPruned()) + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -249,11 +206,11 @@ bool CCoinsViewCache::Flush() { return fOk; } -void CCoinsViewCache::Uncache(const uint256& hash) +void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -262,13 +219,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const -{ - const CCoins* coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; -} - CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const { if (tx.IsCoinBase()) @@ -276,7 +226,7 @@ CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const CAmount nResult = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += GetOutputFor(tx.vin[i]).nValue; + nResult += AccessCoin(tx.vin[i].prevout).out.nValue; return nResult; } @@ -285,9 +235,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -295,44 +243,15 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } -double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const -{ - inChainInputValue = 0; - if (tx.IsCoinBase()) - return 0.0; - double dResult = 0.0; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - { - const CCoins* coins = AccessCoins(txin.prevout.hash); - assert(coins); - if (!coins->IsAvailable(txin.prevout.n)) continue; - if (coins->nHeight <= nHeight) { - dResult += (double)(coins->vout[txin.prevout.n].nValue) * (nHeight-coins->nHeight); - inChainInputValue += coins->vout[txin.prevout.n].nValue; - } - } - return tx.ComputePriority(dResult); -} - -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { - assert(!cache.hasModifier); - cache.hasModifier = true; -} +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. -CCoinsModifier::~CCoinsModifier() +const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { - assert(cache.hasModifier); - cache.hasModifier = false; - it->second.coins.Cleanup(); - cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage - if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { - cache.cacheCoins.erase(it); - } else { - // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + COutPoint iter(txid, 0); + while (iter.n < MAX_OUTPUTS_PER_BLOCK) { + const Coin& alternate = view.AccessCoin(iter); + if (!alternate.IsSpent()) return alternate; + ++iter.n; } -} - -CCoinsViewCursor::~CCoinsViewCursor() -{ + return coinEmpty; } diff --git a/src/coins.h b/src/coins.h index 87fdb3b2f3b..0f70019b1b3 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" @@ -14,140 +15,43 @@ #include "uint256.h" #include +#include #include #include -#include +#include /** - * Pruned version of CTransaction: only retains metadata and unspent transaction outputs + * A UTXO entry. * * Serialized format: - * - VARINT(nVersion) - * - VARINT(nCode) - * - unspentness bitvector, for vout[2] and further; least significant byte first - * - the non-spent CTxOuts (via CTxOutCompressor) - * - VARINT(nHeight) - * - * The nCode value consists of: - * - bit 0: IsCoinBase() - * - bit 1: vout[0] is not spent - * - bit 2: vout[1] is not spent - * - The higher bits encode N, the number of non-zero bytes in the following bitvector. - * - In case both bit 1 and bit 2 are unset, they encode N-1, as there must be at - * least one non-spent output). - * - * Example: 0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e - * <><><--------------------------------------------><----> - * | \ | / - * version code vout[1] height - * - * - version = 1 - * - code = 4 (vout[1] is not spent, and 0 non-zero bytes of bitvector follow) - * - unspentness bitvector: as 0 non-zero bytes follow, it has length 0 - * - vout[1]: 835800816115944e077fe7c803cfa57f29b36bf87c1d35 - * * 8358: compact amount representation for 60000000000 (600 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 816115944e077fe7c803cfa57f29b36bf87c1d35: address uint160 - * - height = 203998 - * - * - * Example: 0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b - * <><><--><--------------------------------------------------><----------------------------------------------><----> - * / \ \ | | / - * version code unspentness vout[4] vout[16] height - * - * - version = 1 - * - code = 9 (coinbase, neither vout[0] or vout[1] are unspent, - * 2 (1, +1 because both bit 1 and bit 2 are unset) non-zero bitvector bytes follow) - * - unspentness bitvector: bits 2 (0x04) and 14 (0x4000) are set, so vout[2+2] and vout[14+2] are unspent - * - vout[4]: 86ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4ee - * * 86ef97d579: compact amount representation for 234925952 (2.35 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 61b01caab50f1b8e9c50a5057eb43c2d9563a4ee: address uint160 - * - vout[16]: bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4 - * * bbd123: compact amount representation for 110397 (0.001 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 8c988f1a4a4de2161e0f50aac7f17e7f9555caa4: address uint160 - * - height = 120891 + * - VARINT((coinbase ? 1 : 0) | (height << 1)) + * - the non-spent CTxOut (via CTxOutCompressor) */ -class CCoins +class Coin { public: - //! whether transaction is a coinbase - bool fCoinBase; + //! unspent transaction output + CTxOut out; - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector vout; + //! whether containing transaction was a coinbase + unsigned int fCoinBase : 1; - //! at which height this transaction was included in the active block chain - int nHeight; + //! at which height this containing transaction was included in the active block chain + uint32_t nHeight : 31; - //! version of the CTransaction; accesses to this value should probably check for nHeight as well, - //! as new tx version will probably only be introduced at certain heights - int nVersion; - - void FromTx(const CTransaction &tx, int nHeightIn) { - fCoinBase = tx.IsCoinBase(); - vout = tx.vout; - nHeight = nHeightIn; - nVersion = tx.nVersion; - ClearUnspendable(); - } - - //! construct a CCoins from a CTransaction, at a given height - CCoins(const CTransaction &tx, int nHeightIn) { - FromTx(tx, nHeightIn); - } + //! construct a Coin from a CTxOut and height/coinbase information. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} void Clear() { + out.SetNull(); fCoinBase = false; - std::vector().swap(vout); nHeight = 0; - nVersion = 0; } //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0), nVersion(0) { } - - //!remove spent outputs at the end of vout - void Cleanup() { - while (vout.size() > 0 && vout.back().IsNull()) - vout.pop_back(); - if (vout.empty()) - std::vector().swap(vout); - } - - void ClearUnspendable() { - BOOST_FOREACH(CTxOut &txout, vout) { - if (txout.scriptPubKey.IsUnspendable()) - txout.SetNull(); - } - Cleanup(); - } - - void swap(CCoins &to) { - std::swap(to.fCoinBase, fCoinBase); - to.vout.swap(vout); - std::swap(to.nHeight, nHeight); - std::swap(to.nVersion, nVersion); - } - - //! equality test - friend bool operator==(const CCoins &a, const CCoins &b) { - // Empty CCoins objects are always equal. - if (a.IsPruned() && b.IsPruned()) - return true; - return a.fCoinBase == b.fCoinBase && - a.nHeight == b.nHeight && - a.nVersion == b.nVersion && - a.vout == b.vout; - } - friend bool operator!=(const CCoins &a, const CCoins &b) { - return !(a == b); - } - - void CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const; + Coin() : fCoinBase(false), nHeight(0) { } bool IsCoinBase() const { return fCoinBase; @@ -155,115 +59,52 @@ class CCoins template void Serialize(Stream &s) const { - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - ::Serialize(s, VARINT(this->nVersion)); - // header code - ::Serialize(s, VARINT(nCode)); - // spentness bitmask - for (unsigned int b = 0; b void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - ::Unserialize(s, VARINT(this->nVersion)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); - } - // coinbase height - ::Unserialize(s, VARINT(nHeight)); - Cleanup(); + uint32_t code = 0; + ::Unserialize(s, VARINT(code)); + nHeight = code >> 1; + fCoinBase = code & 1; + ::Unserialize(s, REF(CTxOutCompressor(out))); } - //! mark a vout spent - bool Spend(uint32_t nPos); - - //! check whether a particular output is still available - bool IsAvailable(unsigned int nPos) const { - return (nPos < vout.size() && !vout[nPos].IsNull()); - } - - //! check whether the entire CCoins is spent - //! note that only !IsPruned() CCoins can be serialized - bool IsPruned() const { - BOOST_FOREACH(const CTxOut &out, vout) - if (!out.IsNull()) - return false; - return true; + bool IsSpent() const { + return out.IsNull(); } size_t DynamicMemoryUsage() const { - size_t ret = memusage::DynamicUsage(vout); - BOOST_FOREACH(const CTxOut &out, vout) { - ret += RecursiveDynamicUsage(out.scriptPubKey); - } - return ret; + return memusage::DynamicUsage(out.scriptPubKey); } }; -class SaltedTxidHasher +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint& id) const { + return SipHashUint256Extra(k0, k1, id.hash, id.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + Coin coin; // The actual cached data. unsigned char flags; enum Flags { @@ -276,21 +117,21 @@ struct CCoinsCacheEntry */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} }; -typedef boost::unordered_map CCoinsMap; +typedef std::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; - /* Don't care about GetKeySize here */ + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -306,17 +147,19 @@ class CCoinsViewCursor class CCoinsView { public: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; + /** Retrieve the Coin (unspent transaction output) for a given outpoint. + * Returns true only when an unspent coin was found, which is returned in coin. + * When false is returned, coin's value is unspecified. + */ + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; + //! Just check whether a given outpoint is unspent. + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -325,6 +168,9 @@ class CCoinsView //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} + + //! Estimate database size (0 if not implemented) + virtual size_t EstimateSize() const { return 0; } }; @@ -336,45 +182,20 @@ class CCoinsViewBacked : public CCoinsView public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + size_t EstimateSize() const override; }; -class CCoinsViewCache; - -/** - * A reference to a mutable cache entry. Encapsulating it allows us to run - * cleanup code after the modification is finished, and keeping track of - * concurrent modifications. - */ -class CCoinsModifier -{ -private: - CCoinsViewCache& cache; - CCoinsMap::iterator it; - size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); - -public: - CCoins* operator->() { return &it->second.coins; } - CCoins& operator*() { return it->second.coins; } - ~CCoinsModifier(); - friend class CCoinsViewCache; -}; - /** CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: - /* Whether this cache has an active modifier. */ - bool hasModifier; - - /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -382,51 +203,48 @@ class CCoinsViewCache : public CCoinsViewBacked mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); - ~CCoinsViewCache(); // Standard CCoinsView methods - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + CCoinsViewCursor* Cursor() const { + throw std::logic_error("CCoinsViewCache cursor iteration not supported."); + } /** - * Check if we have the given tx already loaded in this cache. - * The semantics are the same as HaveCoins(), but no calls to + * Check if we have the given utxo already loaded in this cache. + * The semantics are the same as HaveCoin(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const uint256 &txid) const; + bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a pointer to CCoins in the cache, or NULL if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are + * Return a reference to Coin in the cache, or a pruned one if not found. This is + * more efficient than GetCoin. Modifications to other cache entries are * allowed while accessing the returned pointer. */ - const CCoins* AccessCoins(const uint256 &txid) const; + const Coin& AccessCoin(const COutPoint &output) const; /** - * Return a modifiable reference to a CCoins. If no entry with the given - * txid exists, a new one is created. Simultaneous modifications are not - * allowed. + * Add a coin. Set potential_overwrite to true if a non-pruned version may + * already exist. */ - CCoinsModifier ModifyCoins(const uint256 &txid); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); /** - * Return a modifiable reference to a CCoins. Assumes that no entry with the given - * txid exists and creates a new one. This saves a database access in the case where - * the coins were to be wiped out by FromTx anyway. This should not be called with - * the 2 historical coinbase duplicate pairs because the new coins are marked fresh, and - * in the event the duplicate coinbase was spent before a flush, the now pruned coins - * would not properly overwrite the first coinbase of the pair. Simultaneous modifications - * are not allowed. + * Spend a coin. Pass moveto in order to get the deleted data. + * If no unspent output exists for the passed outpoint, this call + * has no effect. */ - CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); + bool SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** * Push the modifications applied to this cache to its base. @@ -436,12 +254,12 @@ class CCoinsViewCache : public CCoinsViewBacked bool Flush(); /** - * Removes the transaction with the given hash from the cache, if it is + * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ - void Uncache(const uint256 &txid); + void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -460,19 +278,8 @@ class CCoinsViewCache : public CCoinsViewBacked //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - /** - * Return priority of tx at height nHeight. Also calculate the sum of the values of the inputs - * that are already in the chain. These are the inputs that will age and increase priority as - * new blocks are added to the chain. - */ - double GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const; - - const CTxOut &GetOutputFor(const CTxIn& input) const; - - friend class CCoinsModifier; - private: - CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -480,4 +287,13 @@ class CCoinsViewCache : public CCoinsViewBacked CCoinsViewCache(const CCoinsViewCache &); }; +//! Utility function to add all of a transaction's outputs to a cache. +// It assumes that overwrites are only possible for coinbase transactions, +// TODO: pass in a boolean to limit these possible overwrites to known +// (pre-BIP34) cases. +void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); + +//! Utility function to find any unspent output with a given txid. +const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); + #endif // BITCOIN_COINS_H diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 694096237cd..8618dfd7c0a 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -55,11 +55,19 @@ class CDBBatch CDataStream ssKey; CDataStream ssValue; + size_t size_estimate; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + + void Clear() + { + batch.Clear(); + size_estimate = 0; + } template void Write(const K& key, const V& value) @@ -74,6 +82,14 @@ class CDBBatch leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } @@ -86,8 +102,16 @@ class CDBBatch leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } + + size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator @@ -285,7 +309,22 @@ class CDBWrapper * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); + + template + size_t EstimateSize(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; + } }; #endif // BITCOIN_DBWRAPPER_H - diff --git a/src/hash.cpp b/src/hash.cpp index a8218dfaff7..a9109ec9c55 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -211,3 +211,44 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } + +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = (((uint64_t)36) << 56) | extra; + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index eacb8f04fef..b9952d39fc9 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,6 +160,41 @@ class CHashWriter } }; +/** Reads data from an underlying stream, while hashing the read data. */ +template +class CHashVerifier : public CHashWriter +{ +private: + Source* source; + +public: + CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} + + void read(char* pch, size_t nSize) + { + source->read(pch, nSize); + this->write(pch, nSize); + } + + void ignore(size_t nSize) + { + char data[1024]; + while (nSize > 0) { + size_t now = std::min(nSize, 1024); + read(data, now); + nSize -= now; + } + } + + template + CHashVerifier& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) @@ -206,5 +241,6 @@ class CSipHasher * .Finalize() */ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra); #endif // BITCOIN_HASH_H diff --git a/src/init.cpp b/src/init.cpp index 4e1b8964acc..790348a19b2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -152,9 +152,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); @@ -168,7 +168,6 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked // Writes do not need similar protection, as failure to write is handled by the caller. }; -static CCoinsViewDB *pcoinsdbview = NULL; static CCoinsViewErrorCatcher *pcoinscatcher = NULL; static std::unique_ptr globalVerifyHandle; @@ -1503,6 +1502,12 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); + } else { + // If necessary, upgrade from older database format. + if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; + } } if (!LoadBlockIndex(chainparams)) { diff --git a/src/memusage.h b/src/memusage.h index 81e8702954f..b69acafffd1 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -149,7 +151,7 @@ static inline size_t DynamicUsage(const std::shared_ptr& p) // Boost data structures template -struct boost_unordered_node : private X +struct unordered_node : private X { private: void* ptr; @@ -158,13 +160,25 @@ struct boost_unordered_node : private X template static inline size_t DynamicUsage(const boost::unordered_set& s) { - return MallocUsage(sizeof(boost_unordered_node)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); + return MallocUsage(sizeof(unordered_node)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); } template static inline size_t DynamicUsage(const boost::unordered_map& m) { - return MallocUsage(sizeof(boost_unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); + return MallocUsage(sizeof(unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +template +static inline size_t DynamicUsage(const std::unordered_set& s) +{ + return MallocUsage(sizeof(unordered_node)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template +static inline size_t DynamicUsage(const std::unordered_map& m) +{ + return MallocUsage(sizeof(unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 443d5266047..eaad8290b73 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -987,12 +987,11 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 00000000000..42e5be5cc54 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,95 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2024 The Dogecoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include +#include +#include +#include +#include + +//! Magic bytes written at the start of every UTXO snapshot file. +//! Used to detect files that are not UTXO snapshots. +static const std::array SNAPSHOT_MAGIC_BYTES = {{'u', 't', 'x', 'o', 0xff}}; + +//! Current UTXO snapshot file format version. +static const uint16_t SNAPSHOT_VERSION = 1; + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +//! All metadata fields come from an untrusted file, so must be validated +//! before being used. +class SnapshotMetadata +{ +public: + //! The network magic (pchMessageStart) identifying the chain this snapshot + //! belongs to. Used to detect cross-chain load attempts. + uint8_t m_network_magic[4]; + + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! The number of transactions in the chain up to and including the base + //! block; used to estimate IBD progress for the loaded chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { memset(m_network_magic, 0, sizeof(m_network_magic)); } + + SnapshotMetadata( + const uint8_t* network_magic, + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) + { + memcpy(m_network_magic, network_magic, sizeof(m_network_magic)); + } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!ser_action.ForRead()) { + // Serialize: write magic bytes and version first + std::array magic = SNAPSHOT_MAGIC_BYTES; + READWRITE(FLATDATA(magic)); + uint16_t version = SNAPSHOT_VERSION; + READWRITE(version); + READWRITE(FLATDATA(m_network_magic)); + } else { + // Deserialize: read and validate magic bytes and version + std::array magic; + READWRITE(FLATDATA(magic)); + if (magic != SNAPSHOT_MAGIC_BYTES) { + throw std::ios_base::failure( + "Invalid UTXO snapshot magic bytes. " + "Is this really a snapshot file?"); + } + uint16_t version; + READWRITE(version); + if (version != SNAPSHOT_VERSION) { + throw std::ios_base::failure( + "Unsupported UTXO snapshot version"); + } + READWRITE(FLATDATA(m_network_magic)); + } + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index f90d6901697..5426a151443 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -128,7 +128,7 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector > vSolutions; txnouttype whichType; @@ -167,7 +167,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) if (tx.vin[i].scriptWitness.IsNull()) continue; - const CTxOut &prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut &prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; // get the scriptPubKey corresponding to this input: CScript prevScript = prev.scriptPubKey; diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index aadd5bf4df0..97392ad7615 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -790,13 +790,12 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco { COutPoint prevout = txin.prevout; - CCoins prev; - if(pcoinsTip->GetCoins(prevout.hash, prev)) + Coin prev; + if(pcoinsTip->GetCoin(prevout, prev)) { - if (prevout.n < prev.vout.size()) { strHTML += "
  • "; - const CTxOut &vout = prev.vout[prevout.n]; + const CTxOut &vout = prev.out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/rest.cpp b/src/rest.cpp index f3d0a951fe4..ff326d63eb8 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,15 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2022 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chain.h" #include "chainparams.h" +#include "core_io.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "validation.h" #include "httpserver.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" @@ -41,25 +42,27 @@ static const struct { }; struct CCoin { - uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE uint32_t nHeight; CTxOut out; ADD_SERIALIZE_METHODS; + CCoin() : nHeight(0) {} + CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} + template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTxVer); + uint32_t nTxVerDummy = 0; + READWRITE(nTxVerDummy); READWRITE(nHeight); READWRITE(out); } }; -extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); +extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails); extern UniValue mempoolInfoToJSON(); -extern UniValue mempoolToJSON(bool fVerbose = false); +extern UniValue mempoolToJSON(bool fVerbose); extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); @@ -387,7 +390,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, objTx); + TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -462,7 +465,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) std::vector strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); } - // Falls through case RF_BINARY: { try { @@ -513,32 +515,15 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) CCoinsViewCache& viewChain = *pcoinsTip; CCoinsViewMemPool viewMempool(&viewChain, mempool); - if (fCheckMemPool) { - view.SetBackend(viewMempool); // set cache backend to db+mempool in case user likes to query mempool - } else { - view.SetBackend(viewChain); // set cache backend to db only otherwise - } + if (fCheckMemPool) + view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool for (size_t i = 0; i < vOutPoints.size(); i++) { - CCoins coins; - uint256 hash = vOutPoints[i].hash; bool hit = false; - if (view.GetCoins(hash, coins)) { - if (fCheckMemPool) { - mempool.pruneSpent(hash, coins); - } - - if (coins.IsAvailable(vOutPoints[i].n)) { - hit = true; - // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if - // n is valid but points to an already spent output (IsNull). - CCoin coin; - coin.nTxVer = coins.nVersion; - coin.nHeight = coins.nHeight; - coin.out = coins.vout.at(vOutPoints[i].n); - assert(!coin.out.IsNull()); - outs.push_back(coin); - } + Coin coin; + if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + hit = true; + outs.emplace_back(std::move(coin)); } hits.push_back(hit); @@ -582,7 +567,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.pushKV("txvers", (int32_t)coin.nTxVer); utxo.pushKV("height", (int32_t)coin.nHeight); utxo.pushKV("value", ValueFromAmount(coin.out.nValue)); @@ -609,53 +593,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) return true; // continue to process further HTTP reqs on this cxn } -static bool rest_blockhash_by_height(HTTPRequest* req, - const std::string& str_uri_part) - { - if (!CheckWarmup(req)) return false; - std::string height_str; - const RetFormat rf = ParseDataFormat(height_str, str_uri_part); - - int32_t blockheight; - if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { - return RESTERR(req, HTTP_BAD_REQUEST, "Invalid height: " + SanitizeString(height_str)); - } - - CBlockIndex* pblockindex = nullptr; - { - LOCK(cs_main); - if (blockheight > chainActive.Height()) { - return RESTERR(req, HTTP_NOT_FOUND, "Block height out of range"); - } - pblockindex = chainActive[blockheight]; - } - switch (rf) { - case RF_BINARY: { - CDataStream ss_blockhash(SER_NETWORK, PROTOCOL_VERSION); - ss_blockhash << pblockindex->GetBlockHash(); - req->WriteHeader("Content-Type", "application/octet-stream"); - req->WriteReply(HTTP_OK, ss_blockhash.str()); - return true; - } - case RF_HEX: { - req->WriteHeader("Content-Type", "text/plain"); - req->WriteReply(HTTP_OK, pblockindex->GetBlockHash().GetHex() + "\n"); - return true; - } - case RF_JSON: { - req->WriteHeader("Content-Type", "application/json"); - UniValue resp = UniValue(UniValue::VOBJ); - resp.pushKV("blockhash", pblockindex->GetBlockHash().GetHex()); - req->WriteReply(HTTP_OK, resp.write() + "\n"); - return true; - } - default: { - return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); - } - } - } - - static const struct { const char* prefix; bool (*handler)(HTTPRequest* req, const std::string& strReq); @@ -668,7 +605,6 @@ static const struct { {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, - {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; bool StartREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index cfd8b24a95b..2d2fa00285e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,12 +14,15 @@ #include "consensus/validation.h" #include "core_io.h" #include "dogecoin.h" +#include "fs.h" +#include "node/utxo_snapshot.h" #include "validation.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" +#include "txdb.h" #include "txmempool.h" #include "undo.h" #include "util.h" @@ -859,20 +862,36 @@ UniValue getblock(const JSONRPCRequest& request) struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nSerializedSize; - uint256 hashSerialized; - arith_uint256 nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + arith_uint256 nTotalAmount{0}; + uint64_t coins_count{0}; }; +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << VARINT(output.second.out.nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -882,35 +901,125 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - arith_uint256 nTotalAmount = 0; + uint256 prevkey; + std::map outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; iGetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - stats.nSerializedSize += 32 + pcursor->GetValueSize(); - ss << VARINT(0); + prevkey = key.hash; + outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; + stats.nDiskSize = view->EstimateSize(); + return true; +} + +static bool ComputeSnapshotStats( + CAutoFile& afile, + const SnapshotMetadata& metadata, + CCoinsStats& stats, + std::string& error) +{ + stats = CCoinsStats(); + stats.hashBlock = metadata.m_base_blockhash; + + { + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(metadata.m_base_blockhash); + if (it == mapBlockIndex.end()) { + error = "Unable to find snapshot base block"; + return false; + } + stats.nHeight = it->second->nHeight; + } + + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + ss << stats.hashBlock; + + uint256 prevkey; + std::map outputs; + + for (uint64_t coins_loaded = 0; coins_loaded < metadata.m_coins_count; ++coins_loaded) { + try { + COutPoint key; + Coin coin; + afile >> key; + afile >> coin; + + if (coin.nHeight > static_cast(stats.nHeight)) { + error = strprintf("Bad snapshot data after deserializing %d coins", coins_loaded); + return false; + } + if (!MoneyRange(coin.out.nValue)) { + error = strprintf("Bad snapshot data after deserializing %d coins - bad tx out value", coins_loaded); + return false; + } + + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); + } + prevkey = key.hash; + outputs[key.n] = coin; + stats.coins_count++; + } catch (const std::ios_base::failure&) { + error = strprintf("Bad snapshot format or truncated snapshot after deserializing %d coins", coins_loaded); + return false; + } + } + + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } + stats.hashSerialized = ss.GetHash(); + + try { + unsigned char leftover; + afile >> leftover; + error = strprintf("Bad snapshot - coins left over after deserializing %d coins", metadata.m_coins_count); + return false; + } catch (const std::ios_base::failure&) { + } + return true; } +static bool LoadSnapshotCoins(CAutoFile& afile, const SnapshotMetadata& metadata) +{ + CCoinsViewCache snapshot_cache(pcoinsdbview); + + for (uint64_t coins_loaded = 0; coins_loaded < metadata.m_coins_count; ++coins_loaded) { + COutPoint key; + Coin coin; + afile >> key; + afile >> coin; + snapshot_cache.AddCoin(key, std::move(coin), false); + + if ((coins_loaded + 1) % 50000 == 0) { + if (!snapshot_cache.Flush()) { + return false; + } + } + } + + snapshot_cache.SetBestBlock(metadata.m_base_blockhash); + return snapshot_cache.Flush(); +} + UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) @@ -973,8 +1082,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"bytes_serialized\": n, (numeric) The serialized size\n" - " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -986,13 +1095,13 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsTip, stats)) { + if (GetUTXOStats(pcoinsdbview, stats)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); - ret.pushKV("bytes_serialized", (int64_t)stats.nSerializedSize); - ret.pushKV("hash_serialized", stats.hashSerialized.GetHex()); + ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + ret.pushKV("disk_size", (int64_t)stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -1025,7 +1134,6 @@ UniValue gettxout(const JSONRPCRequest& request) " ,...\n" " ]\n" " },\n" - " \"version\" : n, (numeric) The version\n" " \"coinbase\" : true|false (boolean) Coinbase or not\n" "}\n" @@ -1049,33 +1157,32 @@ UniValue gettxout(const JSONRPCRequest& request) if (request.params.size() > 2) fMempool = request.params[2].get_bool(); - CCoins coins; + COutPoint out(hash, n); + Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins)) + if (!view.GetCoin(out, coin) || mempool.isSpent(out)) // TODO: this should be done by the CCoinsViewMemPool return NullUniValue; - mempool.pruneSpent(hash, coins); // TODO: this should be done by the CCoinsViewMemPool } else { - if (!pcoinsTip->GetCoins(hash, coins)) + if (!pcoinsTip->GetCoin(out, coin)) return NullUniValue; } - if (n<0 || (unsigned int)n>=coins.vout.size() || coins.vout[n].IsNull()) + if (coin.IsSpent()) return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); - if ((unsigned int)coins.nHeight == MEMPOOL_HEIGHT) + if (coin.nHeight == MEMPOOL_HEIGHT) ret.pushKV("confirmations", 0); else - ret.pushKV("confirmations", pindex->nHeight - coins.nHeight + 1); - ret.pushKV("value", ValueFromAmount(coins.vout[n].nValue)); + ret.pushKV("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1)); + ret.pushKV("value", ValueFromAmount(coin.out.nValue)); UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); ret.pushKV("scriptPubKey", o); - ret.pushKV("version", coins.nVersion); - ret.pushKV("coinbase", coins.fCoinBase); + ret.pushKV("coinbase", (bool)coin.fCoinBase); return ret; } @@ -1316,7 +1423,7 @@ UniValue getchaintips(const JSONRPCRequest& request) LOCK(cs_main); /* - * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off of them. + * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: * - Make one pass through mapBlockIndex, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. @@ -1769,8 +1876,8 @@ static UniValue getblockstats(const JSONRPCRequest& request) if (loop_inputs) { CAmount tx_total_in = 0; const auto& txundo = blockUndo.vtxundo.at(i - 1); - for (const CTxInUndo& coin: txundo.vprevout) { - const CTxOut& prevoutput = coin.txout; + for (const Coin& coin: txundo.vprevout) { + const CTxOut& prevoutput = coin.out; tx_total_in += prevoutput.nValue; utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; @@ -1854,6 +1961,212 @@ static UniValue getblockstats(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) + throw runtime_error( + "dumptxoutset \"path\"\n" + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest chainstate database to disk.\n" + "\nArguments:\n" + "1. \"path\" (string, required) Path to the output file. If relative, will be prefixed by datadir.\n" + "\nResult:\n" + "{\n" + " \"coins_written\": n, (numeric) The number of coins written in the snapshot\n" + " \"base_hash\": \"hex\", (string) The hash of the base of the snapshot\n" + " \"base_height\": n, (numeric) The height of the base of the snapshot\n" + " \"path\": \"path\", (string) The absolute path to the snapshot file\n" + " \"txoutset_hash\": \"hex\", (string) The hash of the UTXO set contents\n" + " \"nchaintx\": n (numeric) The number of transactions in the chain up to and including the base block\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("dumptxoutset", "\"utxo.dat\"") + + HelpExampleRpc("dumptxoutset", "\"utxo.dat\"") + ); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path) || fs::exists(temppath)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " or its temporary file already exists. Move it out of the way first"); + } + + FILE* file = fsbridge::fopen(temppath, "wb"); + if (file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't open file " + temppath.string() + " for writing"); + } + + CAutoFile afile(file, SER_DISK, CLIENT_VERSION); + std::unique_ptr pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + LOCK(cs_main); + FlushStateToDisk(); + + if (!GetUTXOStats(pcoinsdbview, stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr(pcoinsdbview->Cursor()); + BlockMap::iterator it = mapBlockIndex.find(stats.hashBlock); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to find UTXO base block"); + } + tip = it->second; + } + + SnapshotMetadata metadata(Params().MessageStart(), tip->GetBlockHash(), stats.coins_count, tip->nChainTx); + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter = 0; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } else { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", (int64_t)stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().GetHex()); + result.pushKV("base_height", (int64_t)tip->nHeight); + result.pushKV("path", path.string()); + result.pushKV("txoutset_hash", stats.hashSerialized.GetHex()); + result.pushKV("nchaintx", (int64_t)tip->nChainTx); + return result; +} + +UniValue loadtxoutset(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 2) + throw runtime_error( + "loadtxoutset \"path\" \"txoutset_hash\"\n" + "\nLoad a previously dumped UTXO set from disk into the active chainstate.\n" + "The snapshot base block must already be part of the active chain, and the\n" + "provided txoutset hash must match the snapshot contents.\n" + "\nArguments:\n" + "1. \"path\" (string, required) Path to the snapshot file. If relative, will be prefixed by datadir.\n" + "2. \"txoutset_hash\" (string, required) Expected hash_serialized_2 for the snapshot contents.\n" + "\nResult:\n" + "{\n" + " \"coins_loaded\": n, (numeric) The number of coins loaded from the snapshot\n" + " \"base_hash\": \"hex\", (string) The hash of the base of the snapshot\n" + " \"base_height\": n, (numeric) The height of the base of the snapshot\n" + " \"path\": \"path\" (string) The absolute path to the snapshot file\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("loadtxoutset", "\"utxo.dat\" \"0123456789abcdef\"") + + HelpExampleRpc("loadtxoutset", "\"utxo.dat\", \"0123456789abcdef\"") + ); + + const fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + const uint256 expected_hash = ParseHashV(request.params[1], "txoutset_hash"); + + FILE* file = fsbridge::fopen(path, "rb"); + if (file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't open file " + path.string() + " for reading"); + } + + SnapshotMetadata metadata; + { + CAutoFile afile(file, SER_DISK, CLIENT_VERSION); + try { + afile >> metadata; + } catch (const std::ios_base::failure& e) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Unable to parse metadata: %s", e.what())); + } + } + + if (memcmp(metadata.m_network_magic, Params().MessageStart(), sizeof(metadata.m_network_magic)) != 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Snapshot network does not match this node"); + } + + CBlockIndex* snapshot_index = NULL; + { + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(metadata.m_base_blockhash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Snapshot base block header not found"); + } + snapshot_index = it->second; + if (!chainActive.Contains(snapshot_index)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Snapshot base block must already be part of the active chain"); + } + } + + CCoinsStats snapshot_stats; + std::string snapshot_error; + { + FILE* verify_file = fsbridge::fopen(path, "rb"); + if (verify_file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't reopen file " + path.string() + " for reading"); + } + CAutoFile afile(verify_file, SER_DISK, CLIENT_VERSION); + afile >> metadata; + if (!ComputeSnapshotStats(afile, metadata, snapshot_stats, snapshot_error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, snapshot_error); + } + } + + if (snapshot_stats.hashSerialized != expected_hash) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + strprintf("Snapshot content hash mismatch: expected %s, got %s", + expected_hash.GetHex(), + snapshot_stats.hashSerialized.GetHex())); + } + + CValidationState state; + if (!RewindChainstateToGenesis(state, Params())) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to rewind chainstate: " + FormatStateMessage(state)); + } + + { + FILE* load_file = fsbridge::fopen(path, "rb"); + if (load_file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't reopen file " + path.string() + " for reading"); + } + CAutoFile afile(load_file, SER_DISK, CLIENT_VERSION); + afile >> metadata; + if (!LoadSnapshotCoins(afile, metadata)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to load snapshot coins into chainstate"); + } + } + + if (!ActivateSnapshotTip(state, Params(), snapshot_index)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to activate loaded snapshot: " + FormatStateMessage(state)); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_loaded", (int64_t)metadata.m_coins_count); + result.pushKV("base_hash", snapshot_index->GetBlockHash().GetHex()); + result.pushKV("base_height", (int64_t)snapshot_index->nHeight); + result.pushKV("path", path.string()); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- @@ -1884,6 +2197,8 @@ static const CRPCCommand commands[] = { "hidden", "waitfornewblock", &waitfornewblock, true, {"timeout"} }, { "hidden", "waitforblock", &waitforblock, true, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, true, {"height","timeout"} }, + { "hidden", "dumptxoutset", &dumptxoutset, true, {"path"} }, + { "hidden", "loadtxoutset", &loadtxoutset, true, {"path", "txoutset_hash"} }, }; void RegisterBlockchainRPCCommands(CRPCTable &t) diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 06b417f2275..46531ad7315 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -327,9 +327,10 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins(oneTxid, coins) && coins.nHeight > 0 && coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsSpent() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + pblockindex = chainActive[coin.nHeight]; + } } if (pblockindex == NULL) @@ -736,9 +737,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view BOOST_FOREACH(const CTxIn& txin, mergedTx.vin) { - const uint256& prevHash = txin.prevout.hash; - CCoins coins; - view.AccessCoins(prevHash); // this is certainly allowed to fail + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. } view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long @@ -789,24 +788,26 @@ UniValue signrawtransaction(const JSONRPCRequest& request) if (nOut < 0) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); - vector pkData(ParseHexO(prevOut, "scriptPubKey")); + COutPoint out(txid, nOut); + std::vector pkData(ParseHexO(prevOut, "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { - string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { + std::string err("Previous output scriptPubKey mismatch:\n"); + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and not using the local wallet (private keys @@ -864,13 +865,13 @@ UniValue signrawtransaction(const JSONRPCRequest& request) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (coins == NULL || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -942,9 +943,12 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) nMaxRawTxFee = 0; CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsSpent(); + } bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index f480b3c9ff8..a4eaa1595a0 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,6 +8,7 @@ #include "undo.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" +#include "random.h" #include "validation.h" #include "consensus/validation.h" @@ -16,46 +17,51 @@ #include -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +static uint32_t insecure_rand() { return InsecureRand32(); } + +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace { +//! equality test +bool operator==(const Coin &a, const Coin &b) { + // Empty Coin objects are always equal. + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && + a.nHeight == b.nHeight && + a.out == b.out; +} + class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map map_; + std::map map_; public: - bool GetCoins(const uint256& txid, CCoins& coins) const + bool GetCoin(const COutPoint& outpoint, Coin& coin) const override { - std::map::const_iterator it = map_.find(txid); + std::map::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && InsecureRandBool() == 0) { + coin = it->second; + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256& txid) const - { - CCoins coins; - return GetCoins(txid, coins); - } - - uint256 GetBestBlock() const { return hashBestBlock_; } + uint256 GetBestBlock() const override { return hashBestBlock_; } bool BatchWrite(CCoinsMap& mapCoins, const uint256& hashBlock) { for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && InsecureRandRange(3) == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -71,14 +77,14 @@ class CCoinsViewTest : public CCoinsView class CCoinsViewCacheTest : public CCoinsViewCache { public: - CCoinsViewCacheTest(CCoinsView* base) : CCoinsViewCache(base) {} + CCoinsViewCacheTest(CCoinsView* _base) : CCoinsViewCache(_base) {} void SelfTest() const { // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); } BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -96,7 +102,7 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // This is a large randomized insert/remove simulation test on a variable-size // stack of caches on top of CCoinsViewTest. // -// It will randomly create/update/delete CCoins entries to a tip of caches, with +// It will randomly create/update/delete Coin entries to a tip of caches, with // txids picked from a limited list of random 256-bit hashes. Occasionally, a // new tip is added to the stack of caches, or the tip is flushed and removed. // @@ -114,7 +120,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) bool missed_an_entry = false; // A simple map to track what we expect the cache stack to represent. - std::map result; + std::map result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -125,43 +131,59 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) std::vector txids; txids.resize(NUM_SIMULATION_ITERATIONS / 8); for (unsigned int i = 0; i < txids.size(); i++) { - txids[i] = InsecureRand256(); + txids[i] = GetRandHash(); } for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { // Do a random modification. { - uint256 txid = txids[InsecureRandRange(500) % txids.size()]; // txid we're going to modify in this iteration. - CCoins& coins = result[txid]; - CCoinsModifier entry = stack.back()->ModifyCoins(txid); - BOOST_CHECK(coins == *entry); - if (InsecureRandRange(5) == 0 || coins.IsPruned()) { - if (coins.IsPruned()) { + uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. + Coin& coin = result[COutPoint(txid, 0)]; + + // Determine whether to test HaveCoin before or after Access* (or both). As these functions + // can influence each other's behaviour by pulling things into the cache, all combinations + // are tested. + bool test_havecoin_before = insecure_rand() % 4 == 0; + bool test_havecoin_after = insecure_rand() % 4 == 0; + + bool result_havecoin = test_havecoin_before ? stack.back()->HaveCoin(COutPoint(txid, 0)) : false; + const Coin& entry = stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK(coin == entry); + BOOST_CHECK(!test_havecoin_before || result_havecoin == !entry.IsSpent()); + + if (test_havecoin_after) { + bool ret = stack.back()->HaveCoin(COutPoint(txid, 0)); + BOOST_CHECK(ret == !entry.IsSpent()); + } + + if (insecure_rand() % 5 == 0 || coin.IsSpent()) { + if (coin.IsSpent()) { added_an_entry = true; } else { updated_an_entry = true; } - coins.nVersion = InsecureRand32(); - coins.vout.resize(1); - coins.vout[0].nValue = InsecureRand32(); - *entry = coins; + coin.out.nValue = insecure_rand(); + coin.nHeight = 1; } else { - coins.Clear(); - entry->Clear(); + coin.Clear(); removed_an_entry = true; } + if (coin.IsSpent()) { + stack.back()->SpendCoin(COutPoint(txid, 0)); + } else { + stack.back()->AddCoin(COutPoint(txid, 0), Coin(coin), true); + } } // Once every 1000 iterations and at the end, verify the full cache. - if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - found_an_entry = true; - } else { - BOOST_CHECK(it->second.IsPruned()); + if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { + for (auto it = result.begin(); it != result.end(); it++) { + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(coin == it->second); + if (coin.IsSpent()) { missed_an_entry = true; + } else { + found_an_entry = true; } } BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { @@ -169,22 +191,22 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache - if (stack.size() > 1 && InsecureRandBool() == 0) { - unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (stack.size() > 1 && insecure_rand() % 2 == 0) { + unsigned int flushIndex = insecure_rand() % (stack.size() - 1); stack[flushIndex]->Flush(); } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, change the cache stack. - if (stack.size() > 0 && InsecureRandBool() == 0) { + if (stack.size() > 0 && insecure_rand() % 2 == 0) { //Remove the top cache stack.back()->Flush(); delete stack.back(); stack.pop_back(); } - if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { + if (stack.size() == 0 || (stack.size() < 4 && insecure_rand() % 2)) { //Add a new cache CCoinsView* tip = &base; if (stack.size() > 0) { @@ -216,19 +238,19 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(missed_an_entry); } -typedef std::tuple TxData; // Store of all necessary tx and undo data for next test -std::map alltxs; - -TxData &FindRandomFrom(const std::set &txidset) { - assert(txidset.size()); - std::set::iterator txIt = txidset.lower_bound(InsecureRand256()); - if (txIt == txidset.end()) { - txIt = txidset.begin(); +typedef std::map> UtxoData; +UtxoData utxoData; + +UtxoData::iterator FindRandomFrom(const std::set &utxoSet) { + assert(utxoSet.size()); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(GetRandHash(), 0)); + if (utxoSetIt == utxoSet.end()) { + utxoSetIt = utxoSet.begin(); } - std::map::iterator txdit = alltxs.find(*txIt); - assert(txdit != alltxs.end()); - return txdit->second; + auto utxoDataIt = utxoData.find(*utxoSetIt); + assert(utxoDataIt != utxoData.end()); + return utxoDataIt; } @@ -236,12 +258,12 @@ TxData &FindRandomFrom(const std::set &txidset) { // except the emphasis is on testing the functionality of UpdateCoins // random txs are created and UpdateCoins is used to update the cache stack // In particular it is tested that spending a duplicate coinbase tx -// has the expected effect (the other duplicate is overwritten at all cache levels) +// has the expected effect (the other duplicate is overwitten at all cache levels) BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { bool spent_a_duplicate_coinbase = false; // A simple map to track what we expect the cache stack to represent. - std::map result; + std::map result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -249,13 +271,13 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Track the txids we've used in various sets - std::set coinbaseids; - std::set disconnectedids; - std::set duplicateids; - std::set utxoset; + std::set coinbaseids; + std::set disconnectedids; + std::set duplicateids; + std::set utxoset; for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { - uint32_t randiter = InsecureRand32(); + uint32_t randiter = insecure_rand(); // 19/20 txs add a new transaction if (randiter % 20 < 19) { @@ -263,23 +285,23 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vin.resize(1); tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate - unsigned int height = InsecureRand32(); - CCoins oldcoins; + unsigned int height = insecure_rand(); + Coin old_coin; // 2/20 times create a new coinbase if (randiter % 20 < 2 || coinbaseids.size() < 10) { // 1/10 of those times create a duplicate coinbase - if (InsecureRandRange(10) == 0 && coinbaseids.size()) { - TxData &txd = FindRandomFrom(coinbaseids); + if (insecure_rand() % 10 == 0 && coinbaseids.size()) { + auto utxod = FindRandomFrom(coinbaseids); // Reuse the exact same coinbase - tx = std::get<0>(txd); + tx = std::get<0>(utxod->second); // shouldn't be available for reconnection if its been duplicated - disconnectedids.erase(tx.GetHash()); + disconnectedids.erase(utxod->first); - duplicateids.insert(tx.GetHash()); + duplicateids.insert(utxod->first); } else { - coinbaseids.insert(tx.GetHash()); + coinbaseids.insert(COutPoint(tx.GetHash(), 0)); } assert(CTransaction(tx).IsCoinBase()); } @@ -287,129 +309,120 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // 17/20 times reconnect previous or add a regular tx else { - uint256 prevouthash; + COutPoint prevout; // 1/20 times reconnect a previously disconnected tx if (randiter % 20 == 2 && disconnectedids.size()) { - TxData &txd = FindRandomFrom(disconnectedids); - tx = std::get<0>(txd); - prevouthash = tx.vin[0].prevout.hash; - if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevouthash)) { - disconnectedids.erase(tx.GetHash()); + auto utxod = FindRandomFrom(disconnectedids); + tx = std::get<0>(utxod->second); + prevout = tx.vin[0].prevout; + if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevout)) { + disconnectedids.erase(utxod->first); continue; } // If this tx is already IN the UTXO, then it must be a coinbase, and it must be a duplicate - if (utxoset.count(tx.GetHash())) { + if (utxoset.count(utxod->first)) { assert(CTransaction(tx).IsCoinBase()); - assert(duplicateids.count(tx.GetHash())); + assert(duplicateids.count(utxod->first)); } - disconnectedids.erase(tx.GetHash()); + disconnectedids.erase(utxod->first); } // 16/20 times create a regular tx else { - TxData &txd = FindRandomFrom(utxoset); - prevouthash = std::get<0>(txd).GetHash(); + auto utxod = FindRandomFrom(utxoset); + prevout = utxod->first; // Construct the tx to spend the coins of prevouthash - tx.vin[0].prevout.hash = prevouthash; - tx.vin[0].prevout.n = 0; + tx.vin[0].prevout = prevout; assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevouthash]; + old_coin = result[prevout]; // Update the expected result of prevouthash to know these coins are spent - result[prevouthash].Clear(); + result[prevout].Clear(); - utxoset.erase(prevouthash); + utxoset.erase(prevout); // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) + if (duplicateids.count(prevout)) { spent_a_duplicate_coinbase = true; + } } // Update the expected result to know about the new output coins - result[tx.GetHash()].FromTx(tx, height); + assert(tx.vout.size() == 1); + const COutPoint outpoint(tx.GetHash(), 0); + result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); // Call UpdateCoins on the top cache CTxUndo undo; UpdateCoins(tx, *(stack.back()), undo, height); // Update the utxo set for future spends - utxoset.insert(tx.GetHash()); + utxoset.insert(outpoint); // Track this tx and undo info to use later - alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); - } - - //1/20 times undo a previous transaction - else if (utxoset.size()) { - TxData &txd = FindRandomFrom(utxoset); - - CTransaction &tx = std::get<0>(txd); - CTxUndo &undo = std::get<1>(txd); - CCoins &origcoins = std::get<2>(txd); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,old_coin)); + } else if (utxoset.size()) { + //1/20 times undo a previous transaction + auto utxod = FindRandomFrom(utxoset); - uint256 undohash = tx.GetHash(); + CTransaction &tx = std::get<0>(utxod->second); + CTxUndo &undo = std::get<1>(utxod->second); + Coin &origcoins = std::get<2>(utxod->second); // Update the expected result // Remove new outputs - result[undohash].Clear(); + result[utxod->first].Clear(); // If not coinbase restore prevout if (!tx.IsCoinBase()) { - result[tx.vin[0].prevout.hash] = origcoins; + result[tx.vin[0].prevout] = origcoins; } // Disconnect the tx from the current UTXO // See code in DisconnectBlock // remove outputs - { - CCoinsModifier outs = stack.back()->ModifyCoins(undohash); - outs->Clear(); - } + stack.back()->SpendCoin(utxod->first); // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const CTxInUndo &undoin = undo.vprevout[0]; - ApplyTxInUndo(undoin, *(stack.back()), out); + Coin coin = undo.vprevout[0]; + ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection - disconnectedids.insert(undohash); + disconnectedids.insert(utxod->first); // Update the utxoset - utxoset.erase(undohash); + utxoset.erase(utxod->first); if (!tx.IsCoinBase()) - utxoset.insert(tx.vin[0].prevout.hash); + utxoset.insert(tx.vin[0].prevout); } // Once every 1000 iterations and at the end, verify the full cache. - if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - } else { - BOOST_CHECK(it->second.IsPruned()); - } + if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { + for (auto it = result.begin(); it != result.end(); it++) { + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(coin == it->second); } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache - if (stack.size() > 1 && InsecureRandBool() == 0) { - unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (stack.size() > 1 && insecure_rand() % 2 == 0) { + unsigned int flushIndex = insecure_rand() % (stack.size() - 1); stack[flushIndex]->Flush(); } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, change the cache stack. - if (stack.size() > 0 && InsecureRandBool() == 0) { + if (stack.size() > 0 && insecure_rand() % 2 == 0) { stack.back()->Flush(); delete stack.back(); stack.pop_back(); } - if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { + if (stack.size() == 0 || (stack.size() < 4 && insecure_rand() % 2)) { CCoinsView* tip = &base; if (stack.size() > 0) { tip = stack.back(); @@ -432,53 +445,36 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); - CCoins cc1; + CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + Coin cc1; ss1 >> cc1; - BOOST_CHECK_EQUAL(cc1.nVersion, 1); BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); - BOOST_CHECK_EQUAL(cc1.vout.size(), 2); - BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); - BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); - BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); - BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + BOOST_CHECK_EQUAL(cc1.out.nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); - CCoins cc2; + CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + Coin cc2; ss2 >> cc2; - BOOST_CHECK_EQUAL(cc2.nVersion, 1); BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); - BOOST_CHECK_EQUAL(cc2.vout.size(), 17); - for (int i = 0; i < 17; i++) { - BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); - } - BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); - BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + BOOST_CHECK_EQUAL(cc2.out.nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ssx(SER_DISK, CLIENT_VERSION); - BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); - - CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); - CCoins cc3; + CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + Coin cc3; ss3 >> cc3; - BOOST_CHECK_EQUAL(cc3.nVersion, 0); BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); - BOOST_CHECK_EQUAL(cc3.vout.size(), 1); - BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); - BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); - BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + BOOST_CHECK_EQUAL(cc3.out.nValue, 0); + BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); try { - CCoins cc4; + Coin cc4; ss4 >> cc4; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { @@ -489,16 +485,16 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); - CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { - CCoins cc5; + Coin cc5; ss5 >> cc5; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { } } -const static uint256 TXID; +const static COutPoint OUTPOINT; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -513,15 +509,15 @@ const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; const static auto CLEAN_FLAGS = {char(0), FRESH}; const static auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins& coins) +void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsSpent()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + coin.out.nValue = value; + coin.nHeight = 1; + assert(!coin.IsSpent()); } } @@ -534,25 +530,23 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + SetCoinsValue(value, entry.coin); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coin.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); @@ -580,10 +574,10 @@ class SingleEntryCacheTest CCoinsViewCacheTest cache{&base}; }; -void CheckAccessCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoins(TXID); + test.cache.AccessCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -602,39 +596,39 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Base Cache Result Cache Result * Value Value Value Flags Flags */ - CheckAccessCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); } -void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - SetCoinsValue(modify_value, *test.cache.ModifyCoins(TXID)); + test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -644,79 +638,55 @@ void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_va BOOST_CHECK_EQUAL(result_flags, expected_flags); }; -BOOST_AUTO_TEST_CASE(ccoins_modify) +BOOST_AUTO_TEST_CASE(ccoins_spend) { - /* Check ModifyCoin behavior, requesting a coin from a cache view layered on - * top of a base view, writing a modification to the coin, and then checking + /* Check SpendCoin behavior, requesting a coin from a cache view layered on + * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. * - * Base Cache Write Result Cache Result - * Value Value Value Value Flags Flags + * Base Cache Result Cache Result + * Value Value Value Flags Flags */ - CheckModifyCoins(ABSENT, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(ABSENT, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(PRUNED, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); + CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } -void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); CAmount result_value; char result_flags; try { - SetCoinsValue(modify_value, *test.cache.ModifyNewCoins(TXID, coinbase)); + CTxOut output; + output.nValue = modify_value; + test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + test.cache.SelfTest(); GetCoinsMapEntry(test.cache.map(), result_value, result_flags); } catch (std::logic_error& e) { result_value = FAIL; @@ -727,64 +697,46 @@ void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount mo BOOST_CHECK_EQUAL(result_flags, expected_flags); } -// Simple wrapper for CheckModifyNewCoinsBase function above that loops through +// Simple wrapper for CheckAddCoinBase function above that loops through // different possible base_values, making sure each one gives the same results. -// This wrapper lets the modify_new test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::ModifyNewCoins implementation +// This wrapper lets the coins_add test below be shorter and less repetitive, +// while still verifying that the CoinsViewCache::AddCoin implementation // ignores base values. template -void CheckModifyNewCoins(Args&&... args) +void CheckAddCoin(Args&&... args) { for (CAmount base_value : {ABSENT, PRUNED, VALUE1}) - CheckModifyNewCoinsBase(base_value, std::forward(args)...); + CheckAddCoinBase(base_value, std::forward(args)...); } -BOOST_AUTO_TEST_CASE(ccoins_modify_new) +BOOST_AUTO_TEST_CASE(ccoins_add) { - /* Check ModifyNewCoin behavior, requesting a new coin from a cache view, + /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the ModifyNewCoin coinbase argument set to false, and to true. + * with the AddCoin potential_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result Coinbase - * Value Value Value Flags Flags + * Cache Write Result Cache Result potential_overwrite + * Value Value Value Flags Flags */ - CheckModifyNewCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY , false); - CheckModifyNewCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, 0 , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); } void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d8de765db15..bb7e4732487 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -128,6 +128,23 @@ BOOST_AUTO_TEST_CASE(siphash) tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); + + // Check consistency between CSipHasher and SipHashUint256[Extra]. + FastRandomContext ctx; + for (int i = 0; i < 16; ++i) { + uint64_t k1 = ctx.rand64(); + uint64_t k2 = ctx.rand64(); + uint256 x = GetRandHash(); + uint32_t n = ctx.rand32(); + uint8_t nb[4]; + WriteLE32(nb, n); + CSipHasher sip256(k1, k2); + sip256.Write(x.begin(), 32); + CSipHasher sip288 = sip256; + sip288.Write(nb, 4); + BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); + BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 09a697e407a..c4acb94ee3e 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -112,7 +112,8 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(CCoins(txFrom, 0), txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + const CTxOut& output = txFrom.vout[txTo[i].vin[0].prevout.n]; + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else @@ -316,7 +317,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops)); txFrom.vout[6].nValue = 6000; - coins.ModifyCoins(txFrom.GetHash())->FromTx(txFrom, 0); + AddCoins(coins, txFrom, 0); CMutableTransaction txTo; txTo.vout.resize(1); diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 13d8911f03d..7b1afd71e03 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -102,7 +102,7 @@ void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableT spendingTx.vout[0].nValue = 1; spendingTx.vout[0].scriptPubKey = CScript(); - coins.ModifyCoins(creationTx.GetHash())->FromTx(creationTx, 0); + AddCoins(coins, creationTx, 0); } BOOST_AUTO_TEST_CASE(GetTxSigOpCost) diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index c4983f6f5cc..6dc850b6ab9 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -169,8 +169,8 @@ int main(int argc, char **argv) { try { - CCoins block; - ds >> block; + Coin coin; + ds >> coin; } catch (const std::ios_base::failure& e) {return 0;} break; } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index bf73ee7c20b..07f76303544 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -304,14 +304,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50*CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21*CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22*CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } @@ -467,19 +467,20 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for (int i=0; i<20; i++) threadGroup.create_thread(boost::bind(&CCheckQueue::Thread, boost::ref(scriptcheckqueue))); - CCoins coins; - coins.nVersion = 1; - coins.fCoinBase = false; + std::vector coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { - CTxOut txout; - txout.nValue = 1000; - txout.scriptPubKey = scriptPubKey; - coins.vout.push_back(txout); + Coin coin; + coin.nHeight = 1; + coin.fCoinBase = false; + coin.out.nValue = 1000; + coin.out.scriptPubKey = scriptPubKey; + coins.emplace_back(std::move(coin)); } for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector vChecks; - CScriptCheck check(coins, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); + const CTxOut& output = coins[tx.vin[i].prevout.n].out; + CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); control.Add(vChecks); diff --git a/src/txdb.cpp b/src/txdb.cpp index b54948d5f0f..31fd6ba4fc1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,17 +25,40 @@ static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinEntry { + COutPoint* outpoint; + char key; + CoinEntry(const COutPoint* ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} + + template + void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template + void Unserialize(Stream& s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; + +} CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -50,10 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) + batch.Erase(entry); else - batch.Write(std::make_pair(DB_COINS, it->first), it->second.coins); + batch.Write(entry, it->second.coin); changed++; } count++; @@ -63,10 +88,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return db.WriteBatch(batch); } +size_t CCoinsViewDB::EstimateSize() const +{ + return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); +} + CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } @@ -96,29 +125,31 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record if (i->pcursor->Valid()) { - i->pcursor->GetKey(i->keyTmp); + CoinEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; } else { i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false } return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { - return pcursor->GetValue(coins); + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const @@ -128,14 +159,18 @@ unsigned int CCoinsViewDBCursor::GetValueSize() const bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { @@ -217,3 +252,103 @@ bool CBlockTreeDB::LoadBlockIndexGuts(std::function vout; + + //! at which height this transaction was included in the active block chain + int nHeight; + + //! empty constructor + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } + + template + void Unserialize(Stream &s) { + unsigned int nCode = 0; + // version + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + // header code + ::Unserialize(s, VARINT(nCode)); + fCoinBase = nCode & 1; + std::vector vAvail(2, false); + vAvail[0] = (nCode & 2) != 0; + vAvail[1] = (nCode & 4) != 0; + unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); + // spentness bitmask + while (nMaskCode > 0) { + unsigned char chAvail = 0; + ::Unserialize(s, chAvail); + for (unsigned int p = 0; p < 8; p++) { + bool f = (chAvail & (1 << p)) != 0; + vAvail.push_back(f); + } + if (chAvail != 0) + nMaskCode--; + } + // txouts themself + vout.assign(vAvail.size(), CTxOut()); + for (unsigned int i = 0; i < vAvail.size(); i++) { + if (vAvail[i]) + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); + } + // coinbase height + ::Unserialize(s, VARINT(nHeight)); + } +}; + +} + +/** Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_COINS) { + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + pcursor->Next(); + } else { + break; + } + } + db.WriteBatch(batch); + return true; +} diff --git a/src/txdb.h b/src/txdb.h index b445b0de148..85618e4db7a 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,9 +23,7 @@ class uint256; //! Compensate for extra memory peak (x1.5-x1.9) at flush time. static constexpr int DB_PEAK_USAGE_FACTOR = 2; //! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; -//! Always periodic flush if less than this much space still available. -static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; +static constexpr int MAX_BLOCK_COINSDB_USAGE = 10 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) @@ -74,11 +72,15 @@ class CCoinsViewDB : public CCoinsView public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + + //! Attempt to update from an older database format. Returns whether an error occurred. + bool Upgrade(); + size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ @@ -87,8 +89,8 @@ class CCoinsViewDBCursor: public CCoinsViewCursor public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -98,7 +100,7 @@ class CCoinsViewDBCursor: public CCoinsViewCursor CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr pcursor; - std::pair keyTmp; + std::pair keyTmp; friend class CCoinsViewDB; }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index a97b8805d68..cfcd4e20828 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,32 +1,28 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2018-2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "txmempool.h" -#include "chainparams.h" -#include "clientversion.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "chainparams.h" #include "validation.h" #include "policy/policy.h" #include "policy/fees.h" #include "streams.h" #include "timedata.h" #include "util.h" +#include "clientversion.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, - CAmount _inChainInputValue, - bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): + CAmount _inChainInputValue, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), - inChainInputValue(_inChainInputValue), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) + inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); nModSize = tx->CalculateModifiedSize(GetTxSize()); @@ -35,8 +31,6 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); nModFeesWithDescendants = nFee; - CAmount nValueIn = tx->GetValueOut()+nFee; - assert(inChainInputValue <= nValueIn); feeDelta = 0; @@ -46,14 +40,9 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe nSigOpCostWithAncestors = sigOpCost; } -double -CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const +CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) { - double deltaPriority = ((double)(currentHeight-entryHeight)*inChainInputValue)/nModSize; - double dResult = entryPriority + deltaPriority; - if (dResult < 0) // This should only happen if it was called with a height below entry height - dResult = 0; - return dResult; + *this = other; } void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) @@ -73,6 +62,15 @@ size_t CTxMemPoolEntry::GetTxSize() const return GetVirtualTransactionSize(nTxWeight, sigOpCost); } +double CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const +{ + double deltaPriority = ((double)(currentHeight - entryHeight) * inChainInputValue) / nModSize; + double dResult = entryPriority + deltaPriority; + if (dResult < 0) // This should only happen if it wrapped + return 0.0; + return dResult; +} + // Update the given tx for any in-mempool descendants. // Assumes that setMemPoolChildren is correct for the given tx and all // descendants. @@ -120,7 +118,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendan // vHashesToUpdate is the set of transaction hashes from a disconnected block // which has been re-added to the mempool. -// for each entry, look for descendants that are outside hashesToUpdate, and +// for each entry, look for descendants that are outside vHashesToUpdate, and // add fee/size information for such descendants to the parent. // for each such descendant, also update the ancestor state to include the parent. void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashesToUpdate) @@ -135,7 +133,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashes // accounted for in the state of their ancestors) std::set setAlreadyIncluded(vHashesToUpdate.begin(), vHashesToUpdate.end()); - // Iterate in reverse, so that whenever we are looking at a transaction + // Iterate in reverse, so that whenever we are looking at at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that // setMemPoolChildren will be updated, an assumption made in @@ -346,7 +344,7 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, } CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : - nTransactionsUpdated(0) + nTransactionsUpdated(0), minerPolicyEstimator(new CBlockPolicyEstimator(_minReasonableRelayFee)), ownsEstimator(true) { _clear(); //lock free clear @@ -354,26 +352,30 @@ CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; +} - minerPolicyEstimator = new CBlockPolicyEstimator(_minReasonableRelayFee); +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator), ownsEstimator(false) +{ + _clear(); //lock free clear + + // Sanity checks off by default for performance, because otherwise + // accepting transactions becomes O(N^2) where N is the number + // of transactions in the pool + nCheckFrequency = 0; } CTxMemPool::~CTxMemPool() { - delete minerPolicyEstimator; + if (ownsEstimator) { + delete minerPolicyEstimator; + } } -void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); - - auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); - - // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first->hash == hashTx) { - coins.Spend(it->first->n); // and remove those outputs from coins - it++; - } + return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const @@ -403,9 +405,9 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, // into mapTx. std::map >::const_iterator pos = mapDeltas.find(hash); if (pos != mapDeltas.end()) { - const std::pair &deltas = pos->second; - if (deltas.second) { - mapTx.modify(newit, update_fee_delta(deltas.second)); + const CAmount &delta = pos->second.second; + if (delta) { + mapTx.modify(newit, update_fee_delta(delta)); } } @@ -439,7 +441,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -469,7 +471,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -551,10 +553,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); - int nCoinbaseMaturity = Params().GetConsensus(coins->nHeight).nCoinbaseMaturity; - if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < nCoinbaseMaturity)) { + const Coin &coin = pcoins->AccessCoin(txin.prevout); + if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < Params().GetConsensus(coin.nHeight).nCoinbaseMaturity)) { txToRemove.insert(it); break; } @@ -604,7 +605,7 @@ void CTxMemPool::removeForBlock(const std::vector& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -654,7 +655,6 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const CCoinsViewCache mempoolDuplicate(const_cast(pcoins)); const int64_t nSpendHeight = GetSpendHeight(mempoolDuplicate); - const CChainParams& params = Params(); LOCK(cs); std::list waitingOnDependants; @@ -683,8 +683,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); - assert(coins && coins->IsAvailable(txin.prevout.n)); + assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -736,7 +735,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const else { CValidationState state; bool fCheckResult = tx.IsCoinBase() || - Consensus::CheckTxInputs(params, tx, state, mempoolDuplicate, nSpendHeight); + Consensus::CheckTxInputs(Params(), tx, state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, 1000000); } @@ -752,7 +751,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { bool fCheckResult = entry->GetTx().IsCoinBase() || - Consensus::CheckTxInputs(params, entry->GetTx(), state, mempoolDuplicate, nSpendHeight); + Consensus::CheckTxInputs(Params(), entry->GetTx(), state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); stepsSinceLastRemove = 0; @@ -869,60 +868,53 @@ CFeeRate CTxMemPool::estimateFee(int nBlocks) const LOCK(cs); return minerPolicyEstimator->estimateFee(nBlocks); } + CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const { LOCK(cs); return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); } + double CTxMemPool::estimatePriority(int nBlocks) const { LOCK(cs); return minerPolicyEstimator->estimatePriority(nBlocks); } + double CTxMemPool::estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks) const { LOCK(cs); return minerPolicyEstimator->estimateSmartPriority(nBlocks, answerFoundAtBlocks, *this); } -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const +bool CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const { try { LOCK(cs); fileout << FEEFILE_MIN_BACKCOMPAT_VERSION; - fileout << CLIENT_VERSION; // version that wrote the file + fileout << CLIENT_VERSION; minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { + } catch (const std::exception&) { LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); return false; } return true; } -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) +bool CTxMemPool::ReadFeeEstimates(CAutoFile& filein) { try { int nVersionRequired, nVersionThatWrote; filein >> nVersionRequired >> nVersionThatWrote; if (nVersionRequired > CLIENT_VERSION) return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - - /* From 1.14.7, only accept fee estimate files created by a version equal to or higher than the - * minimum writer version. - * - * If in the future any of the MIN_FEERATE, MAX_FEERATE or FEE_SPACING in policy/fees.h change, - * or these values become configurable, this logic will need to be adapted. */ if (nVersionThatWrote < FEEFILE_MIN_COMPAT_VERSION_WRITER) { return error("CTxMemPool::ReadFeeEstimates(): cannot re-use fee estimate files from version %d, ignoring file (non-fatal)", nVersionThatWrote); } LOCK(cs); minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { + } catch (const std::exception&) { LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); return false; } @@ -987,20 +979,24 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } else { + return false; + } } - return (base->GetCoins(txid, coins) && !coins.IsPruned()); + return (base->GetCoin(outpoint, coin) && !coin.IsSpent()); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins(txid); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1111,7 +1107,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { } } -void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining) { +void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; @@ -1142,18 +1138,18 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRe if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (exists(txin.prevout.hash)) - continue; - auto iter = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || iter->first->hash != txin.prevout.hash) - pvNoSpendsRemaining->push_back(txin.prevout.hash); + if (exists(txin.prevout.hash)) continue; + if (!mapNextTx.count(txin.prevout)) { + pvNoSpendsRemaining->push_back(txin.prevout); + } } } } } - if (maxFeeRateRemoved > CFeeRate(0)) + if (maxFeeRateRemoved > CFeeRate(0)) { LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + } } bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLimit) const { @@ -1162,3 +1158,5 @@ bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLi return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 850d8382ee7..f1c38d85638 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,6 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2018-2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,11 +16,11 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "policy/fees.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" -#undef foreach #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" #include "boost/multi_index/hashed_index.hpp" @@ -31,16 +30,13 @@ class CAutoFile; class CBlockIndex; -/** Minimum client version required to read fee_estimates.dat +/** Minimum client version supported in fee_estimates.dat * Clients with a lower version than this should ignore the file */ static const int FEEFILE_MIN_BACKCOMPAT_VERSION = 1140700; // 1.14.7.0 /** Minimum client version required to read fee_estimates.dat * Files created with a lower version than this are ignored - * - * Due to bucket spacing and min/max changes in 1.14.7, set this - * to 1.14.7.0 exactly. */ static const int FEEFILE_MIN_COMPAT_VERSION_WRITER = 1140700; // 1.14.7.0 @@ -51,13 +47,11 @@ inline double AllowFreeThreshold() inline bool AllowFree(double dPriority) { - // Large (in bytes) low-priority (new, small-coin) transactions - // need a fee. return dPriority > AllowFreeThreshold(); } -/** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ -static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; +/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ +static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { @@ -86,11 +80,6 @@ class CTxMemPool; * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * - * If updating the descendant state is skipped, we can mark the entry as - * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal nTxSize/ - * nFee+feeDelta. (This can potentially happen during a reorg, where we limit the - * amount of work we're willing to do to avoid consuming too much CPU.) - * */ class CTxMemPoolEntry @@ -112,9 +101,7 @@ class CTxMemPoolEntry // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these - // descendants as well. if nCountWithDescendants is 0, treat this entry as - // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be - // correct. + // descendants as well. uint64_t nCountWithDescendants; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) @@ -131,6 +118,8 @@ class CTxMemPoolEntry CAmount _inChainInputValue, bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); + CTxMemPoolEntry(const CTxMemPoolEntry& other); + const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } /** @@ -148,7 +137,7 @@ class CTxMemPoolEntry size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } - // Adjusts the descendant state, if this entry is not dirty. + // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); @@ -360,6 +349,20 @@ enum class MemPoolRemovalReason { REPLACED //! Removed for replacement }; +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -431,14 +434,6 @@ enum class MemPoolRemovalReason { * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * - * Adding transactions from a disconnected block can be very time consuming, - * because we don't have a way to limit the number of in-mempool descendants. - * To bound CPU processing, we limit the amount of work we're willing to do - * to properly update the descendant information for a tx being added from - * a disconnected block. If we would exceed the limit, then we instead mark - * the entry as "dirty", and set the feerate for sorting purposes to be equal - * the feerate of the transaction without any descendants. - * */ class CTxMemPool { @@ -446,6 +441,7 @@ class CTxMemPool uint32_t nCheckFrequency; //!< Value n means that n times in 2^32 we check. unsigned int nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; + bool ownsEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) @@ -530,6 +526,7 @@ class CTxMemPool /** Create a new CTxMemPool. */ CTxMemPool(const CFeeRate& _minReasonableRelayFee); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); ~CTxMemPool(); /** @@ -557,7 +554,7 @@ class CTxMemPool void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector& vtxid); - void pruneSpent(const uint256& hash, CCoins &coins); + bool isSpent(const COutPoint& outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** @@ -585,12 +582,12 @@ class CTxMemPool * new mempool entries may have children in the mempool (which is generally * not the case when otherwise adding transactions). * UpdateTransactionsFromBlock() will find child transactions and update the - * descendant state for each transaction in hashesToUpdate (excluding any - * child transactions present in hashesToUpdate, which are already accounted - * for). Note: hashesToUpdate should be the set of transactions from the + * descendant state for each transaction in vHashesToUpdate (excluding any + * child transactions present in vHashesToUpdate, which are already accounted + * for). Note: vHashesToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ - void UpdateTransactionsFromBlock(const std::vector &hashesToUpdate); + void UpdateTransactionsFromBlock(const std::vector &vHashesToUpdate); /** Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) @@ -618,10 +615,10 @@ class CTxMemPool CFeeRate GetMinFee(size_t sizelimit) const; /** Remove transactions from the mempool until its dynamic size is <= sizelimit. - * pvNoSpendsRemaining, if set, will be populated with the list of transactions + * pvNoSpendsRemaining, if set, will be populated with the list of outpoints * which are not in mempool which no longer have any spends in this mempool. */ - void TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining=NULL); + void TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining=NULL); /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); @@ -647,6 +644,13 @@ class CTxMemPool return (mapTx.count(hash) != 0); } + bool exists(const COutPoint& outpoint) const + { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); + } + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector infoAll() const; @@ -717,7 +721,7 @@ class CTxMemPool void removeUnchecked(txiter entry, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); }; -/** +/** * CCoinsView that brings transactions from a memorypool into view. * It does not check for spendings by memory pool transactions. */ @@ -728,8 +732,8 @@ class CCoinsViewMemPool : public CCoinsViewBacked public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; }; // We want to sort transactions by coin age priority @@ -740,11 +744,9 @@ struct TxCoinAgePriorityCompare bool operator()(const TxCoinAgePriority& a, const TxCoinAgePriority& b) { if (a.first == b.first) - return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); //Reverse order to make sort less than + return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); // Reverse order to make sort less than return a.first < b.first; } }; #endif // BITCOIN_TXMEMPOOL_H - - diff --git a/src/undo.h b/src/undo.h index 7951cc777f5..ed4e40f4dac 100644 --- a/src/undo.h +++ b/src/undo.h @@ -7,58 +7,90 @@ #define BITCOIN_UNDO_H #include "compressor.h" +#include "consensus/consensus.h" #include "primitives/transaction.h" #include "serialize.h" /** Undo information for a CTxIn * - * Contains the prevout's CTxOut being spent, and if this was the - * last output of the affected transaction, its metadata as well - * (coinbase or not, height, transaction version) + * Contains the prevout's CTxOut being spent, and its metadata as well + * (coinbase or not, height). The serialization contains a dummy value of + * zero. This is be compatible with older versions which expect to see + * the transaction version there. */ -class CTxInUndo +class TxInUndoSerializer { -public: - CTxOut txout; // the txout data before being spent - bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase - unsigned int nHeight; // if the outpoint was the last unspent: its height - int nVersion; // if the outpoint was the last unspent: its version - - CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } + const Coin* txout; +public: template void Serialize(Stream &s) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion)); - ::Serialize(s, CTxOutCompressor(REF(txout))); + ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1 : 0))); + if (txout->nHeight > 0) { + // Required to maintain compatibility with older undo format. + ::Serialize(s, (unsigned char)0); + } + ::Serialize(s, CTxOutCompressor(REF(txout->out))); } + TxInUndoSerializer(const Coin* coin) : txout(coin) {} +}; + +class TxInUndoDeserializer +{ + Coin* txout; + +public: template void Unserialize(Stream &s) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - nHeight = nCode / 2; - fCoinBase = nCode & 1; - if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion)); - ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); + txout->nHeight = nCode / 2; + txout->fCoinBase = nCode & 1; + if (txout->nHeight > 0) { + // Old versions stored the version number for the last spend of + // a transaction's outputs. Non-final spends were indicated with + // height = 0. + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + } + ::Unserialize(s, REF(CTxOutCompressor(REF(txout->out)))); } + + TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); + /** Undo information for a CTransaction */ class CTxUndo { public: // undo information for all txins - std::vector vprevout; + std::vector vprevout; - ADD_SERIALIZE_METHODS; + template + void Serialize(Stream& s) const { + // TODO: avoid reimplementing vector serializer + uint64_t count = vprevout.size(); + ::Serialize(s, COMPACTSIZE(REF(count))); + for (const auto& prevout : vprevout) { + ::Serialize(s, REF(TxInUndoSerializer(&prevout))); + } + } - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vprevout); + template + void Unserialize(Stream& s) { + // TODO: avoid reimplementing vector deserializer + uint64_t count = 0; + ::Unserialize(s, COMPACTSIZE(count)); + if (count > MAX_INPUTS_PER_BLOCK) { + throw std::ios_base::failure("Too many input undo records"); + } + vprevout.resize(count); + for (auto& prevout : vprevout) { + ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); + } } }; diff --git a/src/validation.cpp b/src/validation.cpp index e5b5a6e2684..9c3f6e210bf 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include "init.h" #include "policy/fees.h" #include "policy/policy.h" +#include "policy/fees.h" #include "pow.h" #if ENABLE_LIBOQS #include "pqc/pqc_commitment.h" @@ -91,7 +92,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFeeRate = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool(::minRelayTxFeeRate); +CBlockPolicyEstimator mempoolEstimator(::minRelayTxFeeRate); +CTxMemPool mempool(&mempoolEstimator); /** * Returns true if there are nRequired or more blocks of minVersion or above @@ -245,6 +247,7 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc return chain.Genesis(); } +CCoinsViewDB *pcoinsdbview = NULL; CCoinsViewCache *pcoinsTip = NULL; CBlockTreeDB *pblocktree = NULL; @@ -434,15 +437,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -498,7 +501,7 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in unsigned int nSigOps = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; if (prevout.scriptPubKey.IsPayToScriptHash()) nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); } @@ -518,7 +521,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); } return nSigOps; @@ -582,9 +585,9 @@ void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { if (expired != 0) LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); - std::vector vNoSpendsRemaining; + std::vector vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining) pcoinsTip->Uncache(removed); } @@ -611,7 +614,7 @@ static bool IsCurrentForFeeEstimation() bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& coins_to_uncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -704,30 +707,30 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C view.SetBackend(viewMemPool); // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + for (size_t out = 0; out < tx.vout.size(); out++) { + COutPoint outpoint(hash, out); + bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); + if (view.HaveCoin(outpoint)) { + if (!had_coin_in_cache) { + coins_to_uncache.push_back(outpoint); + } + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } } // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) + if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); + } + if (!view.HaveCoin(txin.prevout)) { + if (pfMissingInputs) { *pfMissingInputs = true; + } return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - // Bring the best block into scope view.GetBestBlock(); @@ -762,15 +765,23 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C double nPriorityDummy = 0; pool.ApplyDeltas(hash, nPriorityDummy, nModifiedFees); - CAmount inChainInputValue; - double dPriority = view.GetPriority(tx, chainActive.Height(), inChainInputValue); + CAmount inChainInputValue = 0; + double dPriorityInputs = 0.0; + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (!coin.IsSpent() && coin.nHeight <= chainActive.Height()) { + dPriorityInputs += (double)coin.out.nValue * (chainActive.Height() - coin.nHeight); + inChainInputValue += coin.out.nValue; + } + } + double dPriority = tx.ComputePriority(dPriorityInputs); // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { + const Coin &coin = view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } @@ -1085,10 +1096,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector coins_to_uncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -1140,15 +1151,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus } if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1403,24 +1407,13 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; - } + txundo.vprevout.emplace_back(); + bool is_spent = inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); + assert(is_spent); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1458,22 +1451,22 @@ bool CheckTxInputs(const CChainParams& params, const CTransaction& tx, CValidati for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { + if (coin.IsCoinBase()) { // Dogecoin: Switch maturity at depth 145,000 - int nCoinbaseMaturity = params.GetConsensus(coins->nHeight).nCoinbaseMaturity; - if (nSpendHeight - coins->nHeight < nCoinbaseMaturity) + int nCoinbaseMaturity = params.GetConsensus(coin.nHeight).nCoinbaseMaturity; + if (nSpendHeight - coin.nHeight < nCoinbaseMaturity) return state.Invalid(false, REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); } // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } @@ -1515,11 +1508,19 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + const CScript& scriptPubKey = coin.out.scriptPubKey; + const CAmount amount = coin.out.nValue; // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); + CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); @@ -1531,7 +1532,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, + CScriptCheck check2(scriptPubKey, amount, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); @@ -1561,8 +1562,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << hashBlock; + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1570,10 +1573,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; @@ -1628,37 +1628,41 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // anon namespace +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + view.AddCoin(out, std::move(undo), undo.fCoinBase); - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) @@ -1687,34 +1691,31 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + bool is_spent = view.SpendCoin(out, &coin); + if (!is_spent || tx.vout[o] != coin.out) { + fClean = false; // transaction output mismatch + } + } } // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } @@ -1914,10 +1915,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fEnforceBIP30) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + REJECT_INVALID, "bad-txns-BIP30"); + } + } } } @@ -1984,7 +1987,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { @@ -2128,9 +2131,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), - std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -2171,12 +2173,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) @@ -2257,7 +2259,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), @@ -2753,6 +2755,71 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C return true; } +bool RewindChainstateToGenesis(CValidationState& state, const CChainParams& chainparams) +{ + LOCK(cs_main); + + if (mempool.size() != 0) { + return state.Error("mempool not empty"); + } + + for (CBlockIndex* pindex = chainActive.Tip(); pindex != NULL && pindex->pprev != NULL; pindex = pindex->pprev) { + if ((pindex->nStatus & BLOCK_HAVE_MASK) != BLOCK_HAVE_MASK) { + return state.Error(strprintf("snapshot restore requires block and undo data for block %s", pindex->GetBlockHash().ToString())); + } + } + + while (chainActive.Height() > 0) { + CBlockIndex* pindexDisconnecting = chainActive.Tip(); + if (!DisconnectTip(state, chainparams, true)) { + return error("RewindChainstateToGenesis: unable to disconnect block at height %i", chainActive.Height()); + } + // Re-add disconnected blocks to the candidate set so that CheckBlockIndex + // invariants are satisfied after ActivateSnapshotTip sets a new tip. + // This mirrors what ActivateBestChainStep does when disconnecting blocks + // during a reorg. + setBlockIndexCandidates.insert(pindexDisconnecting); + } + + return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); +} + +bool ActivateSnapshotTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexSnapshot) +{ + CBlockIndex* pindexFork = NULL; + bool fInitialDownload = false; + + { + LOCK(cs_main); + + if (pindexSnapshot == NULL) { + return state.Error("snapshot base block not found"); + } + if (chainActive.Height() != 0) { + return state.Error("chainstate must be rewound to genesis before activating snapshot"); + } + if (!pindexSnapshot->IsValid(BLOCK_VALID_SCRIPTS) || pindexSnapshot->nChainTx == 0) { + return state.Error("snapshot base block must already be fully validated"); + } + + pindexFork = chainActive.Tip(); + pcoinsTip->SetBestBlock(pindexSnapshot->GetBlockHash()); + UpdateTip(pindexSnapshot, chainparams); + setBlockIndexCandidates.insert(pindexSnapshot); + PruneBlockIndexCandidates(); + CheckBlockIndex(chainparams.GetConsensus(chainActive.Height())); + fInitialDownload = IsInitialBlockDownload(); + + if (!FlushStateToDisk(state, FLUSH_STATE_ALWAYS)) { + return false; + } + } + + GetMainSignals().UpdatedBlockTip(pindexSnapshot, pindexFork, fInitialDownload); + uiInterface.NotifyBlockTip(fInitialDownload, pindexSnapshot); + return true; +} + bool ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); diff --git a/src/validation.h b/src/validation.h index 93903ddadfc..63283ebdad6 100644 --- a/src/validation.h +++ b/src/validation.h @@ -15,6 +15,7 @@ #include "chain.h" #include "coins.h" #include "fs.h" +#include "node/utxo_snapshot.h" #include "policy/policy.h" // For RECOMMENDED_MIN_TX_FEE #include "protocol.h" // For CMessageHeader::MessageStartChars #include "script/script_error.h" @@ -40,6 +41,7 @@ class CBlockTreeDB; class CBloomFilter; class CBlockUndo; class CChainParams; +class CCoinsViewDB; class CInv; class CConnman; class CScriptCheck; @@ -461,8 +463,8 @@ class CScriptCheck public: CScriptCheck(): amount(0), ptxTo(0), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR) {} - CScriptCheck(const CCoins& txFromIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : - scriptPubKey(txFromIn.vout[txToIn.vin[nInIn].prevout.n].scriptPubKey), amount(txFromIn.vout[txToIn.vin[nInIn].prevout.n].nValue), + CScriptCheck(const CScript& scriptPubKeyIn, const CAmount amountIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : + scriptPubKey(scriptPubKeyIn), amount(amountIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn) { } bool operator()(); @@ -546,12 +548,21 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn /** Mark a block as invalid. */ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex); +/** Rewind the active chainstate back to genesis. */ +bool RewindChainstateToGenesis(CValidationState& state, const CChainParams& chainparams); + +/** Set the active chain tip to a block whose chainstate has been loaded separately. */ +bool ActivateSnapshotTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexSnapshot); + /** Remove invalidity status from a block and its descendants. */ bool ResetBlockFailureFlags(CBlockIndex *pindex); /** The currently-connected chain of blocks (protected by cs_main). */ extern CChain chainActive; +/** Global variable that points to the coins database (protected by cs_main) */ +extern CCoinsViewDB *pcoinsdbview; + /** Global variable that points to the active CCoinsView (protected by cs_main) */ extern CCoinsViewCache *pcoinsTip;