From c1c85c5e9781cceb8944c82e6d5539ea98e10b48 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 21 Apr 2017 05:26:23 -0700 Subject: [PATCH 01/32] Add support for std::unordered_{map,set} to memusage.h (cherry picked from commit 344a2c4122db5ae309e6dddc99b3e136fd307238) --- src/memusage.h | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/memusage.h b/src/memusage.h index 81e8702954f..b69acafffd1 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include @@ -149,7 +151,7 @@ static inline size_t DynamicUsage(const std::shared_ptr& p) // Boost data structures template -struct boost_unordered_node : private X +struct unordered_node : private X { private: void* ptr; @@ -158,13 +160,25 @@ struct boost_unordered_node : private X template static inline size_t DynamicUsage(const boost::unordered_set& s) { - return MallocUsage(sizeof(boost_unordered_node)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); + return MallocUsage(sizeof(unordered_node)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); } template static inline size_t DynamicUsage(const boost::unordered_map& m) { - return MallocUsage(sizeof(boost_unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); + return MallocUsage(sizeof(unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +template +static inline size_t DynamicUsage(const std::unordered_set& s) +{ + return MallocUsage(sizeof(unordered_node)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template +static inline size_t DynamicUsage(const std::unordered_map& m) +{ + return MallocUsage(sizeof(unordered_node >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } } From 4f0dd597c67f5912083d4eee16730c65bcea6e5c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 21 Apr 2017 05:26:40 -0700 Subject: [PATCH 02/32] Switch CCoinsMap from boost to std unordered_map (cherry picked from commit e6756ad33579766cd73973b7489ef2be9427a4c4) --- src/coins.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coins.h b/src/coins.h index 87fdb3b2f3b..87c3eb54575 100644 --- a/src/coins.h +++ b/src/coins.h @@ -17,7 +17,7 @@ #include #include -#include +#include /** * Pruned version of CTransaction: only retains metadata and unspent transaction outputs @@ -279,7 +279,7 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : coins(), flags(0) {} }; -typedef boost::unordered_map CCoinsMap; +typedef std::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor From 8c2ed80e970d7c2acc2c58c1e0d37ac8d561883c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:04 -0700 Subject: [PATCH 03/32] Add SizeEstimate to CDBBatch This allows estimating the in-memory size of a LevelDB batch. (cherry picked from commit e66dbde6d14cb5f253b3bf8850a98f4fda2d9f49) --- src/dbwrapper.h | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 694096237cd..0990e7b8b0e 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -55,11 +55,19 @@ class CDBBatch CDataStream ssKey; CDataStream ssValue; + size_t size_estimate; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + + void Clear() + { + batch.Clear(); + size_estimate = 0; + } template void Write(const K& key, const V& value) @@ -74,6 +82,14 @@ class CDBBatch leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } @@ -86,8 +102,16 @@ class CDBBatch leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } + + size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator From 3f59bef124c98f9f1b609174983b9ef2bd137d14 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:09 -0700 Subject: [PATCH 04/32] error() in disconnect for disk corruption, not inconsistency The error() function unconditionally reports an error. It should only be used for actually exception situations, and not for the type of inconsistencies that ApplyTxInUndo/DisconnectBlock can graciously deal with. This also makes a subtle semantics change: in ApplyTxInUndo, when a record with metadata is encountered (indicating it is the last spend from a tx), don't wipe the CCoins record if it wasn't empty at that point. This makes sure that UTXO operations never affect any other UTXOs (including those from the same tx). (cherry picked from commit f54580e7e4f225bb615204daef32f72ab8688418) --- src/test/coins_tests.cpp | 2 +- src/validation.cpp | 31 +++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index f480b3c9ff8..68eba46a787 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -16,7 +16,7 @@ #include -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace diff --git a/src/validation.cpp b/src/validation.cpp index e5b5a6e2684..ec915e4f963 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1628,37 +1628,40 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // anon namespace +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + /** * Apply the undo operation of a CTxInUndo to the given chain state. * @param undo The undo object. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; CCoinsModifier coins = view.ModifyCoins(out.hash); if (undo.nHeight != 0) { // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); + if (!coins->IsPruned()) fClean = false; // overwriting existing transaction coins->fCoinBase = undo.fCoinBase; coins->nHeight = undo.nHeight; coins->nVersion = undo.nVersion; } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (coins->IsPruned()) fClean = false; // adding output to missing transaction } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); + if (coins->IsAvailable(out.n)) fClean = false; // overwriting existing output if (coins->vout.size() < out.n+1) coins->vout.resize(out.n+1); coins->vout[out.n] = undo.txout; - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) @@ -1697,8 +1700,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // but it must be corrected before txout nversion ever influences a network rule. if (outsBlock.nVersion < 0) outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); + if (*outs != outsBlock) fClean = false; // transaction mismatch // remove outputs outs->Clear(); @@ -1712,8 +1714,9 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(undo, view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } } } From 8f1caba9ce0e7a546e58c233ed59c97bf1fd092f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:14 -0700 Subject: [PATCH 05/32] Introduce CHashVerifier to hash read data This is necessary later, when we drop the nVersion field from the undo data. At that point deserializing and reserializing the data won't roundtrip anymore, and thus that approach can't be used to verify checksums anymore. With this CHashVerifier approach, we can deserialize while hashing the exact serialized form that was used. This is both more efficient and more correct in that case. (cherry picked from commit e484652fc36ef7135cf08ad380ea7142b6cbadc0) --- src/hash.h | 35 +++++++++++++++++++++++++++++++++++ src/validation.cpp | 9 ++++----- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/hash.h b/src/hash.h index eacb8f04fef..b8de19c0fd4 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,6 +160,41 @@ class CHashWriter } }; +/** Reads data from an underlying stream, while hashing the read data. */ +template +class CHashVerifier : public CHashWriter +{ +private: + Source* source; + +public: + CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} + + void read(char* pch, size_t nSize) + { + source->read(pch, nSize); + this->write(pch, nSize); + } + + void ignore(size_t nSize) + { + char data[1024]; + while (nSize > 0) { + size_t now = std::min(nSize, 1024); + read(data, now); + nSize -= now; + } + } + + template + CHashVerifier& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) diff --git a/src/validation.cpp b/src/validation.cpp index ec915e4f963..cbc837858a3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1561,8 +1561,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << hashBlock; + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1570,10 +1572,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; From ebb2d36100f3ec01bb81dbb274cbab36cc3435f6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:16 -0700 Subject: [PATCH 06/32] Add specialization of SipHash for 256 + 32 bit data We'll need a version of SipHash for tuples of 256 bits and 32 bits data, when CCoinsViewCache switches from using txids to COutPoints as keys. (cherry picked from commit 7e0032290669fae5f52c256856c53038511c7db4) --- src/hash.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ src/hash.h | 1 + src/test/hash_tests.cpp | 17 +++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/src/hash.cpp b/src/hash.cpp index a8218dfaff7..a9109ec9c55 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -211,3 +211,44 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } + +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = (((uint64_t)36) << 56) | extra; + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index b8de19c0fd4..b9952d39fc9 100644 --- a/src/hash.h +++ b/src/hash.h @@ -241,5 +241,6 @@ class CSipHasher * .Finalize() */ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra); #endif // BITCOIN_HASH_H diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d8de765db15..bb7e4732487 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -128,6 +128,23 @@ BOOST_AUTO_TEST_CASE(siphash) tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); + + // Check consistency between CSipHasher and SipHashUint256[Extra]. + FastRandomContext ctx; + for (int i = 0; i < 16; ++i) { + uint64_t k1 = ctx.rand64(); + uint64_t k2 = ctx.rand64(); + uint256 x = GetRandHash(); + uint32_t n = ctx.rand32(); + uint8_t nb[4]; + WriteLE32(nb, n); + CSipHasher sip256(k1, k2); + sip256.Write(x.begin(), 32); + CSipHasher sip288 = sip256; + sip288.Write(nb, 4); + BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); + BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); + } } BOOST_AUTO_TEST_SUITE_END() From a1e51f4964156e7789cd857b30f8798424d2ca9f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:18 -0700 Subject: [PATCH 07/32] Remove/ignore tx version in utxo and undo This makes the following changes: * In undo data and the chainstate database, the transaction nVersion field is removed from the data structures, always written as 0, and ignored when reading. * The definition of hash_serialized in gettxoutsetinfo is changed to no longer incude the nVersion field. It is renamed to hash_serialized_2 to avoid confusion. The new definition also includes transaction height and coinbase information, as this information was missing before. This depends on having a CHashVerifier-based undo data checksum verifier. Apart from changing the definition of serialized_hash, downgrading after using this patch is supported, as no release ever used the value of nVersion field in UTXO entries. (cherry picked from commit d342424301013ec47dc146a4beb49d5c9319d80a) --- qa/rpc-tests/blockchain.py | 3 +-- src/coins.h | 16 +++++----------- src/rest.cpp | 6 ++---- src/rpc/blockchain.cpp | 16 ++++++---------- src/test/coins_tests.cpp | 4 ---- src/test/transaction_tests.cpp | 1 - src/undo.h | 20 ++++++++++++-------- src/validation.cpp | 7 ------- 8 files changed, 26 insertions(+), 47 deletions(-) diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index 7c3b3ad5d49..0ffff2469ec 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -108,9 +108,8 @@ def _test_gettxoutsetinfo(self): assert_equal(res['transactions'], 120) assert_equal(res['height'], 120) assert_equal(res['txouts'], 120) - assert_equal(res['bytes_serialized'], 8520), assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized']), 64) + assert_equal(len(res['hash_serialized_2']), 64) def _test_getblockheader(self): node = self.nodes[0] diff --git a/src/coins.h b/src/coins.h index 87c3eb54575..ffa0882f719 100644 --- a/src/coins.h +++ b/src/coins.h @@ -83,15 +83,10 @@ class CCoins //! at which height this transaction was included in the active block chain int nHeight; - //! version of the CTransaction; accesses to this value should probably check for nHeight as well, - //! as new tx version will probably only be introduced at certain heights - int nVersion; - void FromTx(const CTransaction &tx, int nHeightIn) { fCoinBase = tx.IsCoinBase(); vout = tx.vout; nHeight = nHeightIn; - nVersion = tx.nVersion; ClearUnspendable(); } @@ -104,11 +99,10 @@ class CCoins fCoinBase = false; std::vector().swap(vout); nHeight = 0; - nVersion = 0; } //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0), nVersion(0) { } + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } //!remove spent outputs at the end of vout void Cleanup() { @@ -130,7 +124,6 @@ class CCoins std::swap(to.fCoinBase, fCoinBase); to.vout.swap(vout); std::swap(to.nHeight, nHeight); - std::swap(to.nVersion, nVersion); } //! equality test @@ -140,7 +133,6 @@ class CCoins return true; return a.fCoinBase == b.fCoinBase && a.nHeight == b.nHeight && - a.nVersion == b.nVersion && a.vout == b.vout; } friend bool operator!=(const CCoins &a, const CCoins &b) { @@ -162,7 +154,8 @@ class CCoins assert(fFirst || fSecond || nMaskCode); unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); // version - ::Serialize(s, VARINT(this->nVersion)); + int nVersionDummy = 0; + ::Serialize(s, VARINT(nVersionDummy)); // header code ::Serialize(s, VARINT(nCode)); // spentness bitmask @@ -186,7 +179,8 @@ class CCoins void Unserialize(Stream &s) { unsigned int nCode = 0; // version - ::Unserialize(s, VARINT(this->nVersion)); + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); // header code ::Unserialize(s, VARINT(nCode)); fCoinBase = nCode & 1; diff --git a/src/rest.cpp b/src/rest.cpp index f3d0a951fe4..2f7138993ca 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -41,7 +41,6 @@ static const struct { }; struct CCoin { - uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE uint32_t nHeight; CTxOut out; @@ -50,7 +49,8 @@ struct CCoin { template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTxVer); + uint32_t nTxVerDummy = 0; + READWRITE(nTxVerDummy); READWRITE(nHeight); READWRITE(out); } @@ -533,7 +533,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if // n is valid but points to an already spent output (IsNull). CCoin coin; - coin.nTxVer = coins.nVersion; coin.nHeight = coins.nHeight; coin.out = coins.vout.at(vOutPoints[i].n); assert(!coin.out.IsNull()); @@ -582,7 +581,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.pushKV("txvers", (int32_t)coin.nTxVer); utxo.pushKV("height", (int32_t)coin.nHeight); utxo.pushKV("value", ValueFromAmount(coin.out.nValue)); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index cfd8b24a95b..030d8d9522c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -863,11 +863,10 @@ struct CCoinsStats uint256 hashBlock; uint64_t nTransactions; uint64_t nTransactionOutputs; - uint64_t nSerializedSize; uint256 hashSerialized; arith_uint256 nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nTotalAmount(0) {} }; //! Calculate statistics about the unspent transaction output set @@ -890,16 +889,17 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { stats.nTransactions++; ss << key; + ss << VARINT(coins.nHeight * 2 + coins.fCoinBase); for (unsigned int i=0; iGetValueSize(); ss << VARINT(0); } else { return error("%s: unable to read value", __func__); @@ -973,8 +973,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"bytes_serialized\": n, (numeric) The serialized size\n" - " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -991,8 +990,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); - ret.pushKV("bytes_serialized", (int64_t)stats.nSerializedSize); - ret.pushKV("hash_serialized", stats.hashSerialized.GetHex()); + ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -1025,7 +1023,6 @@ UniValue gettxout(const JSONRPCRequest& request) " ,...\n" " ]\n" " },\n" - " \"version\" : n, (numeric) The version\n" " \"coinbase\" : true|false (boolean) Coinbase or not\n" "}\n" @@ -1074,7 +1071,6 @@ UniValue gettxout(const JSONRPCRequest& request) UniValue o(UniValue::VOBJ); ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); ret.pushKV("scriptPubKey", o); - ret.pushKV("version", coins.nVersion); ret.pushKV("coinbase", coins.fCoinBase); return ret; diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 68eba46a787..2657e15f068 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -141,7 +141,6 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } else { updated_an_entry = true; } - coins.nVersion = InsecureRand32(); coins.vout.resize(1); coins.vout[0].nValue = InsecureRand32(); *entry = coins; @@ -435,7 +434,6 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); CCoins cc1; ss1 >> cc1; - BOOST_CHECK_EQUAL(cc1.nVersion, 1); BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); BOOST_CHECK_EQUAL(cc1.vout.size(), 2); @@ -448,7 +446,6 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); CCoins cc2; ss2 >> cc2; - BOOST_CHECK_EQUAL(cc2.nVersion, 1); BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); BOOST_CHECK_EQUAL(cc2.vout.size(), 17); @@ -467,7 +464,6 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); CCoins cc3; ss3 >> cc3; - BOOST_CHECK_EQUAL(cc3.nVersion, 0); BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); BOOST_CHECK_EQUAL(cc3.vout.size(), 1); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index bf73ee7c20b..e14dc041ba6 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -468,7 +468,6 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { threadGroup.create_thread(boost::bind(&CCheckQueue::Thread, boost::ref(scriptcheckqueue))); CCoins coins; - coins.nVersion = 1; coins.fCoinBase = false; for(uint32_t i = 0; i < mtx.vin.size(); i++) { CTxOut txout; diff --git a/src/undo.h b/src/undo.h index 7951cc777f5..37703a8913d 100644 --- a/src/undo.h +++ b/src/undo.h @@ -14,7 +14,8 @@ * * Contains the prevout's CTxOut being spent, and if this was the * last output of the affected transaction, its metadata as well - * (coinbase or not, height, transaction version) + * (coinbase or not, height). Earlier versions also stored the transaction + * version. */ class CTxInUndo { @@ -22,16 +23,17 @@ class CTxInUndo CTxOut txout; // the txout data before being spent bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase unsigned int nHeight; // if the outpoint was the last unspent: its height - int nVersion; // if the outpoint was the last unspent: its version - CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } + CTxInUndo() : txout(), fCoinBase(false), nHeight(0) {} + CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) { } template void Serialize(Stream &s) const { ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion)); + if (nHeight > 0) { + int nVersionDummy = 0; + ::Serialize(s, VARINT(nVersionDummy)); + } ::Serialize(s, CTxOutCompressor(REF(txout))); } @@ -41,8 +43,10 @@ class CTxInUndo ::Unserialize(s, VARINT(nCode)); nHeight = nCode / 2; fCoinBase = nCode & 1; - if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion)); + if (nHeight > 0) { + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + } ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); } }; diff --git a/src/validation.cpp b/src/validation.cpp index cbc837858a3..db728bbc498 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1415,7 +1415,6 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund CTxInUndo& undo = txundo.vprevout.back(); undo.nHeight = coins->nHeight; undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; } } } @@ -1651,7 +1650,6 @@ int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& if (!coins->IsPruned()) fClean = false; // overwriting existing transaction coins->fCoinBase = undo.fCoinBase; coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; } else { if (coins->IsPruned()) fClean = false; // adding output to missing transaction } @@ -1694,11 +1692,6 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI outs->ClearUnspendable(); CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; if (*outs != outsBlock) fClean = false; // transaction mismatch // remove outputs From fc752c4355a6a08ddda7387c8647a7102a14bc52 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 12 May 2017 15:19:19 -0700 Subject: [PATCH 08/32] Report on-disk size in gettxoutsetinfo (cherry picked from commit c3aa0c11947dfd82702df276d39bb7f748dd83a1) --- qa/rpc-tests/blockchain.py | 1 + src/coins.cpp | 1 + src/coins.h | 4 ++++ src/dbwrapper.h | 17 ++++++++++++++++- src/rpc/blockchain.cpp | 7 ++++++- src/txdb.cpp | 5 +++++ src/txdb.h | 2 ++ 7 files changed, 35 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/blockchain.py b/qa/rpc-tests/blockchain.py index 0ffff2469ec..18ddc8dbfab 100755 --- a/qa/rpc-tests/blockchain.py +++ b/qa/rpc-tests/blockchain.py @@ -108,6 +108,7 @@ def _test_gettxoutsetinfo(self): assert_equal(res['transactions'], 120) assert_equal(res['height'], 120) assert_equal(res['txouts'], 120) + assert_greater_than(res['disk_size'], 0) assert_equal(len(res['bestblock']), 64) assert_equal(len(res['hash_serialized_2']), 64) diff --git a/src/coins.cpp b/src/coins.cpp index cda0ea5bd3b..83712f6b968 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -55,6 +55,7 @@ uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } +size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} diff --git a/src/coins.h b/src/coins.h index ffa0882f719..fc07d7a5bb5 100644 --- a/src/coins.h +++ b/src/coins.h @@ -319,6 +319,9 @@ class CCoinsView //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} + + //! Estimate database size (0 if not implemented) + virtual size_t EstimateSize() const { return 0; } }; @@ -336,6 +339,7 @@ class CCoinsViewBacked : public CCoinsView void SetBackend(CCoinsView &viewIn); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); CCoinsViewCursor *Cursor() const; + size_t EstimateSize() const override; }; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 0990e7b8b0e..8618dfd7c0a 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -309,7 +309,22 @@ class CDBWrapper * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); + + template + size_t EstimateSize(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; + } }; #endif // BITCOIN_DBWRAPPER_H - diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 030d8d9522c..8369a3ea9a0 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -865,8 +865,10 @@ struct CCoinsStats uint64_t nTransactionOutputs; uint256 hashSerialized; arith_uint256 nTotalAmount; + uint64_t nDiskSize; + arith_uint256 nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nDiskSize(0), nTotalAmount(0) {} }; //! Calculate statistics about the unspent transaction output set @@ -908,6 +910,7 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } stats.hashSerialized = ss.GetHash(); stats.nTotalAmount = nTotalAmount; + stats.nDiskSize = view->EstimateSize(); return true; } @@ -974,6 +977,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -991,6 +995,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.pushKV("transactions", (int64_t)stats.nTransactions); ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + ret.pushKV("disk_size", (int64_t)stats.nDiskSize); ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); diff --git a/src/txdb.cpp b/src/txdb.cpp index b54948d5f0f..c56935ee936 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -67,6 +67,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return db.WriteBatch(batch); } +size_t CCoinsViewDB::EstimateSize() const +{ + return db.EstimateSize(DB_COINS, (char)(DB_COINS+1)); +} + CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { } diff --git a/src/txdb.h b/src/txdb.h index b445b0de148..c523d12c886 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -79,6 +79,8 @@ class CCoinsViewDB : public CCoinsView uint256 GetBestBlock() const; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); CCoinsViewCursor *Cursor() const; + + size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ From 8798688f57e8720f01e9aa5a253a2818fd54f2a6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:19 -0700 Subject: [PATCH 09/32] Store/allow tx metadata in all undo records Previously, transaction metadata (height, coinbase or not, and before the previous commit also nVersion) was only stored for undo records that correspond to the last output of a transaction being spent. This only saves 2 bytes per undo record. Change this to storing this information for every undo record, and stop complaining for having it in non-last output spends. This means that undo dat written with this patch won't be readable by older versions anymore. (cherry picked from commit 7d991b55dbf0b0f6e21c0680ee3ebd09df09012f) --- src/undo.h | 3 +-- src/validation.cpp | 17 ++++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/undo.h b/src/undo.h index 37703a8913d..35ddc856d0a 100644 --- a/src/undo.h +++ b/src/undo.h @@ -12,8 +12,7 @@ /** Undo information for a CTxIn * - * Contains the prevout's CTxOut being spent, and if this was the - * last output of the affected transaction, its metadata as well + * Contains the prevout's CTxOut being spent, and its metadata as well * (coinbase or not, height). Earlier versions also stored the transaction * version. */ diff --git a/src/validation.cpp b/src/validation.cpp index db728bbc498..caa5e49e2d6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1411,11 +1411,9 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund // mark an outpoint spent, and construct undo information txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - } + CTxInUndo& undo = txundo.vprevout.back(); + undo.nHeight = coins->nHeight; + undo.fCoinBase = coins->fCoinBase; } } // add outputs @@ -1646,11 +1644,16 @@ int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& CCoinsModifier coins = view.ModifyCoins(out.hash); if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) fClean = false; // overwriting existing transaction + if (!coins->IsPruned()) { + if (coins->fCoinBase != undo.fCoinBase || (uint32_t)coins->nHeight != undo.nHeight) fClean = false; // metadata mismatch + } + // restore height/coinbase tx metadata from undo data coins->fCoinBase = undo.fCoinBase; coins->nHeight = undo.nHeight; } else { + // Undo data does not contain height/coinbase. This should never happen + // for newly created undo entries. Previously, this data was only saved + // for the last spend of a transaction's outputs, so check IsPruned(). if (coins->IsPruned()) fClean = false; // adding output to missing transaction } if (coins->IsAvailable(out.n)) fClean = false; // overwriting existing output From 246da0fc94f7a0ba4097c731036b74663cc6ab02 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:23 -0700 Subject: [PATCH 10/32] Introduce Coin, a single unspent output (cherry picked from commit 422634e2f5ac1ff74cd358144cecbac63007adc4) --- src/coins.h | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/coins.h b/src/coins.h index fc07d7a5bb5..d04bd3f4a3e 100644 --- a/src/coins.h +++ b/src/coins.h @@ -19,6 +19,68 @@ #include #include +/** + * A UTXO entry. + * + * Serialized format: + * - VARINT((coinbase ? 1 : 0) | (height << 1)) + * - the non-spent CTxOut (via CTxOutCompressor) + */ +class Coin +{ +public: + //! whether the containing transaction was a coinbase + bool fCoinBase; + + //! unspent transaction output + CTxOut out; + + //! at which height the containing transaction was included in the active block chain + uint32_t nHeight; + + //! construct a Coin from a CTxOut and height/coinbase properties. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : fCoinBase(fCoinBaseIn), out(std::move(outIn)), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : fCoinBase(fCoinBaseIn), out(outIn), nHeight(nHeightIn) {} + + void Clear() { + out.SetNull(); + fCoinBase = false; + nHeight = 0; + } + + //! empty constructor + Coin() : fCoinBase(false), nHeight(0) { } + + bool IsCoinBase() const { + return fCoinBase; + } + + template + void Serialize(Stream &s) const { + assert(!IsPruned()); + uint32_t code = nHeight * 2 + fCoinBase; + ::Serialize(s, VARINT(code)); + ::Serialize(s, CTxOutCompressor(REF(out))); + } + + template + void Unserialize(Stream &s) { + uint32_t code = 0; + ::Unserialize(s, VARINT(code)); + nHeight = code >> 1; + fCoinBase = code & 1; + ::Unserialize(s, REF(CTxOutCompressor(out))); + } + + bool IsPruned() const { + return out.IsNull(); + } + + size_t DynamicMemoryUsage() const { + return memusage::DynamicUsage(out.scriptPubKey); + } +}; + /** * Pruned version of CTransaction: only retains metadata and unspent transaction outputs * From af9be44f5e6760deb78be445da0bbc83f3dbfe3e Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:25 -0700 Subject: [PATCH 11/32] Replace CTxInUndo with Coin The earlier CTxInUndo class now holds the same information as the Coin class. Instead of duplicating functionality, replace CTxInUndo with a serialization adapter for Coin. (cherry picked from commit cb2c7fdac2dc74368ed24ae4717ed72178956b92) --- src/test/coins_tests.cpp | 4 +-- src/undo.h | 77 +++++++++++++++++++++++++++------------- src/validation.cpp | 18 +++++----- 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 2657e15f068..89bf06cfe8a 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -16,7 +16,7 @@ #include -int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(const Coin& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace @@ -370,7 +370,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const CTxInUndo &undoin = undo.vprevout[0]; + const Coin &undoin = undo.vprevout[0]; ApplyTxInUndo(undoin, *(stack.back()), out); } // Store as a candidate for reconnection diff --git a/src/undo.h b/src/undo.h index 35ddc856d0a..ed4e40f4dac 100644 --- a/src/undo.h +++ b/src/undo.h @@ -7,61 +7,90 @@ #define BITCOIN_UNDO_H #include "compressor.h" +#include "consensus/consensus.h" #include "primitives/transaction.h" #include "serialize.h" /** Undo information for a CTxIn * * Contains the prevout's CTxOut being spent, and its metadata as well - * (coinbase or not, height). Earlier versions also stored the transaction - * version. + * (coinbase or not, height). The serialization contains a dummy value of + * zero. This is be compatible with older versions which expect to see + * the transaction version there. */ -class CTxInUndo +class TxInUndoSerializer { -public: - CTxOut txout; // the txout data before being spent - bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase - unsigned int nHeight; // if the outpoint was the last unspent: its height - - CTxInUndo() : txout(), fCoinBase(false), nHeight(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) { } + const Coin* txout; +public: template void Serialize(Stream &s) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) { - int nVersionDummy = 0; - ::Serialize(s, VARINT(nVersionDummy)); + ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1 : 0))); + if (txout->nHeight > 0) { + // Required to maintain compatibility with older undo format. + ::Serialize(s, (unsigned char)0); } - ::Serialize(s, CTxOutCompressor(REF(txout))); + ::Serialize(s, CTxOutCompressor(REF(txout->out))); } + TxInUndoSerializer(const Coin* coin) : txout(coin) {} +}; + +class TxInUndoDeserializer +{ + Coin* txout; + +public: template void Unserialize(Stream &s) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - nHeight = nCode / 2; - fCoinBase = nCode & 1; - if (nHeight > 0) { + txout->nHeight = nCode / 2; + txout->fCoinBase = nCode & 1; + if (txout->nHeight > 0) { + // Old versions stored the version number for the last spend of + // a transaction's outputs. Non-final spends were indicated with + // height = 0. int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); + ::Unserialize(s, REF(CTxOutCompressor(REF(txout->out)))); } + + TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); + /** Undo information for a CTransaction */ class CTxUndo { public: // undo information for all txins - std::vector vprevout; + std::vector vprevout; - ADD_SERIALIZE_METHODS; + template + void Serialize(Stream& s) const { + // TODO: avoid reimplementing vector serializer + uint64_t count = vprevout.size(); + ::Serialize(s, COMPACTSIZE(REF(count))); + for (const auto& prevout : vprevout) { + ::Serialize(s, REF(TxInUndoSerializer(&prevout))); + } + } - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vprevout); + template + void Unserialize(Stream& s) { + // TODO: avoid reimplementing vector deserializer + uint64_t count = 0; + ::Unserialize(s, COMPACTSIZE(count)); + if (count > MAX_INPUTS_PER_BLOCK) { + throw std::ios_base::failure("Too many input undo records"); + } + vprevout.resize(count); + for (auto& prevout : vprevout) { + ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); + } } }; diff --git a/src/validation.cpp b/src/validation.cpp index caa5e49e2d6..f9a5a971854 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1409,11 +1409,9 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) assert(false); // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; + txundo.vprevout.emplace_back(coins->vout[nPos], coins->nHeight, coins->fCoinBase); + bool ret = coins->Spend(nPos); + assert(ret); } } // add outputs @@ -1632,13 +1630,13 @@ enum DisconnectResult }; /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. * @return A DisconnectResult as an int */ -int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(const Coin& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; @@ -1659,7 +1657,7 @@ int ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& if (coins->IsAvailable(out.n)) fClean = false; // overwriting existing output if (coins->vout.size() < out.n+1) coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + coins->vout[out.n] = undo.out; return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } @@ -1708,7 +1706,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI return error("DisconnectBlock(): transaction and undo data inconsistent"); for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; + const Coin &undo = txundo.vprevout[j]; int res = ApplyTxInUndo(undo, view, out); if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; fClean = fClean && res != DISCONNECT_UNCLEAN; From bc6957ac612f103644e5aecaf4c4441c7dc5688d Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:27 -0700 Subject: [PATCH 12/32] Optimization: Coin&& to ApplyTxInUndo This avoids a prevector copy in ApplyTxInUndo. (cherry picked from commit bd83111a0fcfdb97204a0180bcf861d3b53bb6c2) --- src/test/coins_tests.cpp | 6 +++--- src/validation.cpp | 16 +++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 89bf06cfe8a..02ea1c15b14 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -16,7 +16,7 @@ #include -int ApplyTxInUndo(const Coin& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace @@ -370,8 +370,8 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const Coin &undoin = undo.vprevout[0]; - ApplyTxInUndo(undoin, *(stack.back()), out); + Coin coin = undo.vprevout[0]; + ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection disconnectedids.insert(undohash); diff --git a/src/validation.cpp b/src/validation.cpp index f9a5a971854..5dea89ba49e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1636,7 +1636,7 @@ enum DisconnectResult * @param out The out point that corresponds to the tx input. * @return A DisconnectResult as an int */ -int ApplyTxInUndo(const Coin& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; @@ -1657,7 +1657,7 @@ int ApplyTxInUndo(const Coin& undo, CCoinsViewCache& view, const COutPoint& out) if (coins->IsAvailable(out.n)) fClean = false; // overwriting existing output if (coins->vout.size() < out.n+1) coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.out; + coins->vout[out.n] = std::move(undo.out); return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } @@ -1701,16 +1701,18 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const Coin &undo = txundo.vprevout[j]; - int res = ApplyTxInUndo(undo, view, out); + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } From 2f09e11d7dd8d16e453a2fc0e2b70e17ce13d915 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:29 -0700 Subject: [PATCH 13/32] Introduce new per-txout CCoinsViewCache functions The new functions are: * CCoinsViewCache::AddCoin: Add a single COutPoint/Coin pair. * CCoinsViewCache::SpendCoin: Remove a single COutPoint. * AddCoins: utility function that invokes CCoinsViewCache::AddCoin for each output in a CTransaction. * AccessByTxid: utility function that searches for any output with a given txid. * CCoinsViewCache::AccessCoin: retrieve the Coin for a COutPoint. * CCoinsViewCache::HaveCoins: check whether a non-empty Coin exists for a given COutPoint. The AddCoin and SpendCoin methods will eventually replace ModifyCoins and ModifyNewCoins, AddCoins will replace CCoins::FromTx, and the new AccessCoins and HaveCoins functions will replace their per-txid counterparts. Note that AccessCoin for now returns a copy of the Coin object. In a later commit it will be change to returning a const reference (which keeps working in all call sites). (cherry picked from commit 000391132608343c66488d62625c714814959bc9) --- src/coins.cpp | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/coins.h | 33 +++++++++++++++++++- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index 83712f6b968..42d3c48e074 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,6 +4,7 @@ #include "coins.h" +#include "consensus/consensus.h" #include "memusage.h" #include "random.h" @@ -70,7 +71,7 @@ size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { +CCoinsMap::iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { CCoinsMap::iterator it = cacheCoins.find(txid); if (it != cacheCoins.end()) return it; @@ -153,6 +154,58 @@ CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbas return CCoinsModifier(*this, ret.first, 0); } +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { + assert(!coin.IsPruned()); + if (coin.out.scriptPubKey.IsUnspendable()) return; + CCoinsMap::iterator it; + bool inserted; + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint.hash), std::tuple<>()); + bool fresh = false; + if (!inserted) { + cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + } + if (!possible_overwrite) { + if (it->second.coins.IsAvailable(outpoint.n)) { + throw std::logic_error("Adding new coin that replaces non-pruned entry"); + } + fresh = it->second.coins.IsPruned() && !(it->second.flags & CCoinsCacheEntry::DIRTY); + } + if (it->second.coins.vout.size() <= outpoint.n) { + it->second.coins.vout.resize(outpoint.n + 1); + } + it->second.coins.vout[outpoint.n] = std::move(coin.out); + it->second.coins.nHeight = coin.nHeight; + it->second.coins.fCoinBase = coin.fCoinBase; + it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); + cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); +} + +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { + bool fCoinbase = tx.IsCoinBase(); + const uint256& txid = tx.GetHash(); + for (size_t i = 0; i < tx.vout.size(); ++i) { + // Pass fCoinbase as the possible_overwrite flag to AddCoin, in order to correctly + // deal with the pre-BIP30 occurrances of duplicate coinbase transactions. + cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), fCoinbase); + } +} + +void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoins(outpoint.hash); + if (it == cacheCoins.end()) return; + cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + if (moveout && it->second.coins.IsAvailable(outpoint.n)) { + *moveout = Coin(it->second.coins.vout[outpoint.n], it->second.coins.nHeight, it->second.coins.fCoinBase); + } + it->second.coins.Spend(outpoint.n); // Ignore return value: SpendCoin has no effect if no UTXO found. + if (it->second.coins.IsPruned() && it->second.flags & CCoinsCacheEntry::FRESH) { + cacheCoins.erase(it); + } else { + cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + it->second.flags |= CCoinsCacheEntry::DIRTY; + } +} + const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { CCoinsMap::const_iterator it = FetchCoins(txid); if (it == cacheCoins.end()) { @@ -162,6 +215,18 @@ const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { } } +static const Coin coinEmpty; + +const Coin CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoins(outpoint.hash); + if (it == cacheCoins.end() || !it->second.coins.IsAvailable(outpoint.n)) { + return coinEmpty; + } else { + return Coin(it->second.coins.vout[outpoint.n], it->second.coins.nHeight, it->second.coins.fCoinBase); + } +} + + bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { CCoinsMap::const_iterator it = FetchCoins(txid); // We're using vtx.empty() instead of IsPruned here for performance reasons, @@ -171,6 +236,11 @@ bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { return (it != cacheCoins.end() && !it->second.coins.vout.empty()); } +bool CCoinsViewCache::HaveCoins(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoins(outpoint.hash); + return (it != cacheCoins.end() && it->second.coins.IsAvailable(outpoint.n)); +} + bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { CCoinsMap::const_iterator it = cacheCoins.find(txid); return it != cacheCoins.end(); @@ -337,3 +407,16 @@ CCoinsModifier::~CCoinsModifier() CCoinsViewCursor::~CCoinsViewCursor() { } + +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. + +const Coin AccessByTxid(const CCoinsViewCache& view, const uint256& txid) +{ + COutPoint iter(txid, 0); + while (iter.n < MAX_OUTPUTS_PER_BLOCK) { + const Coin& alternate = view.AccessCoin(iter); + if (!alternate.IsPruned()) return alternate; + ++iter.n; + } + return coinEmpty; +} diff --git a/src/coins.h b/src/coins.h index d04bd3f4a3e..bc9d380829c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -452,6 +452,7 @@ class CCoinsViewCache : public CCoinsViewBacked // Standard CCoinsView methods bool GetCoins(const uint256 &txid, CCoins &coins) const; bool HaveCoins(const uint256 &txid) const; + bool HaveCoins(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -470,6 +471,14 @@ class CCoinsViewCache : public CCoinsViewBacked */ const CCoins* AccessCoins(const uint256 &txid) const; + /** + * Return a copy of a Coin in the cache, or a pruned one if not found. This is + * more efficient than GetCoins. Modifications to other cache entries are + * allowed while accessing the returned pointer. + * TODO: return a reference to a Coin after changing CCoinsViewCache storage. + */ + const Coin AccessCoin(const COutPoint &output) const; + /** * Return a modifiable reference to a CCoins. If no entry with the given * txid exists, a new one is created. Simultaneous modifications are not @@ -488,6 +497,19 @@ class CCoinsViewCache : public CCoinsViewBacked */ CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); + /** + * Add a coin. Set potential_overwrite to true if a non-pruned version may + * already exist. + */ + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); + + /** + * Spend a coin. Pass moveto in order to get the deleted data. + * If no unspent output exists for the passed outpoint, this call + * has no effect. + */ + void SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); + /** * Push the modifications applied to this cache to its base. * Failure to call this method before destruction will cause the changes to be forgotten. @@ -532,7 +554,7 @@ class CCoinsViewCache : public CCoinsViewBacked friend class CCoinsModifier; private: - CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoins(const uint256 &txid) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -540,4 +562,13 @@ class CCoinsViewCache : public CCoinsViewBacked CCoinsViewCache(const CCoinsViewCache &); }; +//! Utility function to add all of a transaction's outputs to a cache. +// It assumes that overwrites are only possible for coinbase transactions, +// TODO: pass in a boolean to limit these possible overwrites to known +// (pre-BIP34) cases. +void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); + +//! Utility function to find any unspent output with a given txid. +const Coin AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); + #endif // BITCOIN_COINS_H From 31a15d30b6d740c98c907af6adddc40d79239984 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:30 -0700 Subject: [PATCH 14/32] Switch from per-tx to per-txout CCoinsViewCache methods in some places (cherry picked from commit f68cdfe92b37f5a75be612b7de3c1a03245696d0) --- src/bench/ccoins_caching.cpp | 4 +- src/bitcoin-tx.cpp | 26 +++++----- src/rpc/rawtransaction.cpp | 41 ++++++++-------- src/test/script_P2SH_tests.cpp | 2 +- src/test/sigopcount_tests.cpp | 2 +- src/test/transaction_tests.cpp | 4 +- src/validation.cpp | 87 ++++++++++++++-------------------- 7 files changed, 78 insertions(+), 88 deletions(-) diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1e8e3d462fc..5aab3381fd2 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50 * CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21 * CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22 * CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 0a70e930314..15f788bf15e 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -576,24 +576,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout must be positive"); + COutPoint out(txid, nOut); std::vector pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsPruned() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw std::runtime_error(err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]); + newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and private keys given, @@ -615,13 +617,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (!coins || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsPruned()) { fComplete = false; continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 06b417f2275..0459b3902bf 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -736,9 +736,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view BOOST_FOREACH(const CTxIn& txin, mergedTx.vin) { - const uint256& prevHash = txin.prevout.hash; - CCoins coins; - view.AccessCoins(prevHash); // this is certainly allowed to fail + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. } view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long @@ -789,24 +787,26 @@ UniValue signrawtransaction(const JSONRPCRequest& request) if (nOut < 0) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); - vector pkData(ParseHexO(prevOut, "scriptPubKey")); + COutPoint out(txid, nOut); + std::vector pkData(ParseHexO(prevOut, "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { - string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + const Coin& coin = view.AccessCoin(out); + if (!coin.IsPruned() && coin.out.scriptPubKey != scriptPubKey) { + std::string err("Previous output scriptPubKey mismatch:\n"); + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and not using the local wallet (private keys @@ -864,13 +864,13 @@ UniValue signrawtransaction(const JSONRPCRequest& request) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (coins == NULL || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsPruned()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -942,9 +942,12 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) nMaxRawTxFee = 0; CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsPruned(); + } bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 09a697e407a..be64dc1845f 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -316,7 +316,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops)); txFrom.vout[6].nValue = 6000; - coins.ModifyCoins(txFrom.GetHash())->FromTx(txFrom, 0); + AddCoins(coins, txFrom, 0); CMutableTransaction txTo; txTo.vout.resize(1); diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 13d8911f03d..7b1afd71e03 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -102,7 +102,7 @@ void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableT spendingTx.vout[0].nValue = 1; spendingTx.vout[0].scriptPubKey = CScript(); - coins.ModifyCoins(creationTx.GetHash())->FromTx(creationTx, 0); + AddCoins(coins, creationTx, 0); } BOOST_AUTO_TEST_CASE(GetTxSigOpCost) diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index e14dc041ba6..d4919fd17a0 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -304,14 +304,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50*CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21*CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22*CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/validation.cpp b/src/validation.cpp index 5dea89ba49e..12aaa904d41 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -769,8 +769,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { + const Coin &coin = view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } @@ -1140,15 +1140,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus } if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsPruned()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1403,19 +1396,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.emplace_back(coins->vout[nPos], coins->nHeight, coins->fCoinBase); - bool ret = coins->Spend(nPos); - assert(ret); + txundo.vprevout.emplace_back(); + inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1640,24 +1626,21 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - if (!coins->IsPruned()) { - if (coins->fCoinBase != undo.fCoinBase || (uint32_t)coins->nHeight != undo.nHeight) fClean = false; // metadata mismatch + if (view.HaveCoins(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsPruned()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata } - // restore height/coinbase tx metadata from undo data - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - } else { - // Undo data does not contain height/coinbase. This should never happen - // for newly created undo entries. Previously, this data was only saved - // for the last spend of a transaction's outputs, so check IsPruned(). - if (coins->IsPruned()) fClean = false; // adding output to missing transaction } - if (coins->IsAvailable(out.n)) fClean = false; // overwriting existing output - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = std::move(undo.out); + view.AddCoin(out, std::move(undo), undo.fCoinBase); return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } @@ -1688,15 +1671,15 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - if (*outs != outsBlock) fClean = false; // transaction mismatch - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + view.SpendCoin(out, &coin); + if (tx.vout[o] != coin.out) { + fClean = false; // transaction output mismatch + } + } } // restore inputs @@ -1912,10 +1895,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fEnforceBIP30) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoins(COutPoint(tx->GetHash(), o))) { + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + REJECT_INVALID, "bad-txns-BIP30"); + } + } } } @@ -1982,7 +1967,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { From 7d65b9b28f4bad931a8eb3245c6f4bbddf38784a Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 27 Apr 2017 10:37:33 -0400 Subject: [PATCH 15/32] Only pass things committed to by tx's witness hash to CScriptCheck This clarifies a bit more the ways in which the new script execution cache could break consensus in the future if additional data from the CCoins object were to be used as a part of script execution. After this change, any such consensus breaks should be very visible to reviewers, hopefully ensuring no such changes can be made. (cherry picked from commit c87b957a32e03c09d410abadf661f87eb813bcdb) --- src/test/script_P2SH_tests.cpp | 3 ++- src/test/transaction_tests.cpp | 3 ++- src/validation.cpp | 12 ++++++++++-- src/validation.h | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index be64dc1845f..c4acb94ee3e 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -112,7 +112,8 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(CCoins(txFrom, 0), txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + const CTxOut& output = txFrom.vout[txTo[i].vin[0].prevout.n]; + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index d4919fd17a0..6943f22fe9f 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -478,7 +478,8 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector vChecks; - CScriptCheck check(coins, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); + const CTxOut& output = coins.vout[tx.vin[i].prevout.n]; + CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); control.Add(vChecks); diff --git a/src/validation.cpp b/src/validation.cpp index 12aaa904d41..d364b29b4b0 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1499,8 +1499,16 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi const CCoins* coins = inputs.AccessCoins(prevout.hash); assert(coins); + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + const CScript& scriptPubKey = coins->vout[prevout.n].scriptPubKey; + const CAmount amount = coins->vout[prevout.n].nValue; + // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); + CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); @@ -1512,7 +1520,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, + CScriptCheck check2(scriptPubKey, amount, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); diff --git a/src/validation.h b/src/validation.h index 93903ddadfc..6bad0e7f489 100644 --- a/src/validation.h +++ b/src/validation.h @@ -461,8 +461,8 @@ class CScriptCheck public: CScriptCheck(): amount(0), ptxTo(0), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR) {} - CScriptCheck(const CCoins& txFromIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : - scriptPubKey(txFromIn.vout[txToIn.vin[nInIn].prevout.n].scriptPubKey), amount(txFromIn.vout[txToIn.vin[nInIn].prevout.n].nValue), + CScriptCheck(const CScript& scriptPubKeyIn, const CAmount amountIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : + scriptPubKey(scriptPubKeyIn), amount(amountIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn) { } bool operator()(); From f9ce8af68fdd45e6f70cb86d5d9a27d375371304 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:32 -0700 Subject: [PATCH 16/32] Switch CScriptCheck to use Coin instead of CCoins (cherry picked from commit 8b3868c1b4bf89c41b26ecb3a4b7c3a2557e3868) --- src/test/transaction_tests.cpp | 15 ++++++++------- src/validation.cpp | 8 ++++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 6943f22fe9f..07f76303544 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -467,18 +467,19 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for (int i=0; i<20; i++) threadGroup.create_thread(boost::bind(&CCheckQueue::Thread, boost::ref(scriptcheckqueue))); - CCoins coins; - coins.fCoinBase = false; + std::vector coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { - CTxOut txout; - txout.nValue = 1000; - txout.scriptPubKey = scriptPubKey; - coins.vout.push_back(txout); + Coin coin; + coin.nHeight = 1; + coin.fCoinBase = false; + coin.out.nValue = 1000; + coin.out.scriptPubKey = scriptPubKey; + coins.emplace_back(std::move(coin)); } for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector vChecks; - const CTxOut& output = coins.vout[tx.vin[i].prevout.n]; + const CTxOut& output = coins[tx.vin[i].prevout.n].out; CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); diff --git a/src/validation.cpp b/src/validation.cpp index d364b29b4b0..8709b956626 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1496,16 +1496,16 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsPruned()); // We very carefully only pass in things to CScriptCheck which // are clearly committed to by tx' witness hash. This provides // a sanity check that our caching is not introducing consensus // failures through additional data in, eg, the coins being // spent being checked as a part of CScriptCheck. - const CScript& scriptPubKey = coins->vout[prevout.n].scriptPubKey; - const CAmount amount = coins->vout[prevout.n].nValue; + const CScript& scriptPubKey = coin.out.scriptPubKey; + const CAmount amount = coin.out.nValue; // Verify signature CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); From 5db4293d89de6d5d63bc2e81ff04a0d0cde5dcd9 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:34 -0700 Subject: [PATCH 17/32] Switch tests from ModifyCoins to AddCoin/SpendCoin (cherry picked from commit 961e4839793f8b4ad37d29672faf1695ff6ec03a) --- src/test/coins_tests.cpp | 268 +++++++++++++++++---------------------- 1 file changed, 119 insertions(+), 149 deletions(-) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 02ea1c15b14..dcde4f1ded3 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -21,6 +21,15 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund namespace { +//! equality test +bool operator==(const Coin &a, const Coin &b) { + // Empty Coin objects are always equal. + if (a.IsPruned() && b.IsPruned()) return true; + return a.fCoinBase == b.fCoinBase && + a.nHeight == b.nHeight && + a.out == b.out; +} + class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; @@ -133,8 +142,9 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) { uint256 txid = txids[InsecureRandRange(500) % txids.size()]; // txid we're going to modify in this iteration. CCoins& coins = result[txid]; - CCoinsModifier entry = stack.back()->ModifyCoins(txid); - BOOST_CHECK(coins == *entry); + const Coin& entry = stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK((entry.IsPruned() && coins.IsPruned()) || entry == Coin(coins.vout[0], coins.nHeight, coins.fCoinBase)); + if (InsecureRandRange(5) == 0 || coins.IsPruned()) { if (coins.IsPruned()) { added_an_entry = true; @@ -143,12 +153,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } coins.vout.resize(1); coins.vout[0].nValue = InsecureRand32(); - *entry = coins; } else { coins.Clear(); - entry->Clear(); removed_an_entry = true; } + if (coins.IsPruned()) { + stack.back()->SpendCoin(COutPoint(txid, 0)); + } else { + stack.back()->AddCoin(COutPoint(txid, 0), Coin(coins.vout[0], coins.nHeight, coins.fCoinBase), true); + } } // Once every 1000 iterations and at the end, verify the full cache. @@ -324,8 +337,9 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) + if (duplicateids.count(prevouthash)) { spent_a_duplicate_coinbase = true; + } } // Update the expected result to know about the new output coins @@ -340,10 +354,8 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // Track this tx and undo info to use later alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); - } - - //1/20 times undo a previous transaction - else if (utxoset.size()) { + } else if (utxoset.size()) { + //1/20 times undo a previous transaction TxData &txd = FindRandomFrom(utxoset); CTransaction &tx = std::get<0>(txd); @@ -364,8 +376,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // See code in DisconnectBlock // remove outputs { - CCoinsModifier outs = stack.back()->ModifyCoins(undohash); - outs->Clear(); + stack.back()->SpendCoin(COutPoint(undohash, 0)); } // restore inputs if (!tx.IsCoinBase()) { @@ -495,6 +506,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) } const static uint256 TXID; +const static COutPoint OUTPOINT = {uint256(), 0}; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -576,10 +588,10 @@ class SingleEntryCacheTest CCoinsViewCacheTest cache{&base}; }; -void CheckAccessCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoins(TXID); + test.cache.AccessCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -598,39 +610,39 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Base Cache Result Cache Result * Value Value Value Flags Flags */ - CheckAccessCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); } -void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - SetCoinsValue(modify_value, *test.cache.ModifyCoins(TXID)); + test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -640,79 +652,55 @@ void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_va BOOST_CHECK_EQUAL(result_flags, expected_flags); }; -BOOST_AUTO_TEST_CASE(ccoins_modify) +BOOST_AUTO_TEST_CASE(ccoins_spend) { - /* Check ModifyCoin behavior, requesting a coin from a cache view layered on - * top of a base view, writing a modification to the coin, and then checking + /* Check SpendCoin behavior, requesting a coin from a cache view layered on + * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. * - * Base Cache Write Result Cache Result - * Value Value Value Value Flags Flags + * Base Cache Result Cache Result + * Value Value Value Flags Flags */ - CheckModifyCoins(ABSENT, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(ABSENT, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(PRUNED, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); + CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } -void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); CAmount result_value; char result_flags; try { - SetCoinsValue(modify_value, *test.cache.ModifyNewCoins(TXID, coinbase)); + CTxOut output; + output.nValue = modify_value; + test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + test.cache.SelfTest(); GetCoinsMapEntry(test.cache.map(), result_value, result_flags); } catch (std::logic_error& e) { result_value = FAIL; @@ -723,64 +711,46 @@ void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount mo BOOST_CHECK_EQUAL(result_flags, expected_flags); } -// Simple wrapper for CheckModifyNewCoinsBase function above that loops through +// Simple wrapper for CheckAddCoinBase function above that loops through // different possible base_values, making sure each one gives the same results. -// This wrapper lets the modify_new test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::ModifyNewCoins implementation +// This wrapper lets the coins_add test below be shorter and less repetitive, +// while still verifying that the CoinsViewCache::AddCoin implementation // ignores base values. template -void CheckModifyNewCoins(Args&&... args) +void CheckAddCoin(Args&&... args) { for (CAmount base_value : {ABSENT, PRUNED, VALUE1}) - CheckModifyNewCoinsBase(base_value, std::forward(args)...); + CheckAddCoinBase(base_value, std::forward(args)...); } -BOOST_AUTO_TEST_CASE(ccoins_modify_new) +BOOST_AUTO_TEST_CASE(ccoins_add) { - /* Check ModifyNewCoin behavior, requesting a new coin from a cache view, + /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the ModifyNewCoin coinbase argument set to false, and to true. + * with the AddCoin potential_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result Coinbase - * Value Value Value Flags Flags + * Cache Write Result Cache Result potential_overwrite + * Value Value Value Flags Flags */ - CheckModifyNewCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY , false); - CheckModifyNewCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, 0 , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); } void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) From b91f50dbae71e53c6b690f50284421e2f46790f5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:36 -0700 Subject: [PATCH 18/32] Remove ModifyCoins/ModifyNewCoins (cherry picked from commit 05293f3cb75ad08ca23cba8e795e27d4d5e4d690) --- src/coins.cpp | 84 +-------------------------------------------------- src/coins.h | 47 ---------------------------- 2 files changed, 1 insertion(+), 130 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index 42d3c48e074..ba56dbfd5ef 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -60,12 +60,7 @@ size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } - -CCoinsViewCache::~CCoinsViewCache() -{ - assert(!hasModifier); -} +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) { } size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; @@ -98,62 +93,6 @@ bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { - assert(!hasModifier); - std::pair ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - size_t cachedCoinUsage = 0; - if (ret.second) { - if (!base->GetCoins(txid, ret.first->second.coins)) { - // The parent view does not have this entry; mark it as fresh. - ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } else if (ret.first->second.coins.IsPruned()) { - // The parent view only has a pruned entry for this; mark it as fresh. - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } - } else { - cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); - } - // Assume that whenever ModifyCoins is called, the entry will be modified. - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, cachedCoinUsage); -} - -/* ModifyNewCoins allows for faster coin modification when creating the new - * outputs from a transaction. It assumes that BIP 30 (no duplicate txids) - * applies and has already been tested for (or the test is not required due to - * BIP 34, height in coinbase). If we can assume BIP 30 then we know that any - * non-coinbase transaction we are adding to the UTXO must not already exist in - * the utxo unless it is fully spent. Thus we can check only if it exists DIRTY - * at the current level of the cache, in which case it is not safe to mark it - * FRESH (b/c then its spentness still needs to flushed). If it's not dirty and - * doesn't exist or is pruned in the current cache, we know it either doesn't - * exist or is pruned in parent caches, which is the definition of FRESH. The - * exception to this is the two historical violations of BIP 30 in the chain, - * both of which were coinbases. We do not mark these fresh so we we can ensure - * that they will still be properly overwritten when spent. - */ -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { - assert(!hasModifier); - std::pair ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - if (!coinbase) { - // New coins must not already exist. - if (!ret.first->second.coins.IsPruned()) - throw std::logic_error("ModifyNewCoins should not find pre-existing coins on a non-coinbase unless they are pruned!"); - - if (!(ret.first->second.flags & CCoinsCacheEntry::DIRTY)) { - // If the coin is known to be pruned (have no unspent outputs) in - // the current view and the cache entry is not dirty, we know the - // coin also must be pruned in the parent view as well, so it is safe - // to mark this fresh. - ret.first->second.flags |= CCoinsCacheEntry::FRESH; - } - } - ret.first->second.coins.Clear(); - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, 0); -} - void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { assert(!coin.IsPruned()); if (coin.out.scriptPubKey.IsUnspendable()) return; @@ -257,7 +196,6 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). CCoinsMap::iterator itUs = cacheCoins.find(it->first); @@ -384,26 +322,6 @@ double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, CAmount } return tx.ComputePriority(dResult); } - -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { - assert(!cache.hasModifier); - cache.hasModifier = true; -} - -CCoinsModifier::~CCoinsModifier() -{ - assert(cache.hasModifier); - cache.hasModifier = false; - it->second.coins.Cleanup(); - cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage - if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { - cache.cacheCoins.erase(it); - } else { - // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); - } -} - CCoinsViewCursor::~CCoinsViewCursor() { } diff --git a/src/coins.h b/src/coins.h index bc9d380829c..0a5def1697e 100644 --- a/src/coins.h +++ b/src/coins.h @@ -405,36 +405,10 @@ class CCoinsViewBacked : public CCoinsView }; -class CCoinsViewCache; - -/** - * A reference to a mutable cache entry. Encapsulating it allows us to run - * cleanup code after the modification is finished, and keeping track of - * concurrent modifications. - */ -class CCoinsModifier -{ -private: - CCoinsViewCache& cache; - CCoinsMap::iterator it; - size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); - -public: - CCoins* operator->() { return &it->second.coins; } - CCoins& operator*() { return it->second.coins; } - ~CCoinsModifier(); - friend class CCoinsViewCache; -}; - /** CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: - /* Whether this cache has an active modifier. */ - bool hasModifier; - - /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -447,7 +421,6 @@ class CCoinsViewCache : public CCoinsViewBacked public: CCoinsViewCache(CCoinsView *baseIn); - ~CCoinsViewCache(); // Standard CCoinsView methods bool GetCoins(const uint256 &txid, CCoins &coins) const; @@ -479,24 +452,6 @@ class CCoinsViewCache : public CCoinsViewBacked */ const Coin AccessCoin(const COutPoint &output) const; - /** - * Return a modifiable reference to a CCoins. If no entry with the given - * txid exists, a new one is created. Simultaneous modifications are not - * allowed. - */ - CCoinsModifier ModifyCoins(const uint256 &txid); - - /** - * Return a modifiable reference to a CCoins. Assumes that no entry with the given - * txid exists and creates a new one. This saves a database access in the case where - * the coins were to be wiped out by FromTx anyway. This should not be called with - * the 2 historical coinbase duplicate pairs because the new coins are marked fresh, and - * in the event the duplicate coinbase was spent before a flush, the now pruned coins - * would not properly overwrite the first coinbase of the pair. Simultaneous modifications - * are not allowed. - */ - CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); - /** * Add a coin. Set potential_overwrite to true if a non-pruned version may * already exist. @@ -551,8 +506,6 @@ class CCoinsViewCache : public CCoinsViewBacked const CTxOut &GetOutputFor(const CTxIn& input) const; - friend class CCoinsModifier; - private: CCoinsMap::iterator FetchCoins(const uint256 &txid) const; From 01250abbee6eebcdd1e70248da46ce218abd533f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:37 -0700 Subject: [PATCH 19/32] Replace CCoins-based CTxMemPool::pruneSpent with isSpent (cherry picked from commit 13870b56fcd0bfacedce3ae42a3de3d5e9dc7bc1) --- src/rest.cpp | 6 +----- src/rpc/blockchain.cpp | 3 +-- src/txmempool.cpp | 11 ++--------- src/txmempool.h | 2 +- 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/rest.cpp b/src/rest.cpp index 2f7138993ca..4bafe6bb7d3 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -524,11 +524,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) uint256 hash = vOutPoints[i].hash; bool hit = false; if (view.GetCoins(hash, coins)) { - if (fCheckMemPool) { - mempool.pruneSpent(hash, coins); - } - - if (coins.IsAvailable(vOutPoints[i].n)) { + if (coins.IsAvailable(vOutPoints[i].n) && (!fCheckMemPool || !mempool.isSpent(vOutPoints[i]))) { hit = true; // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if // n is valid but points to an already spent output (IsNull). diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 8369a3ea9a0..7b933bbb551 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1055,9 +1055,8 @@ UniValue gettxout(const JSONRPCRequest& request) if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins)) + if (!view.GetCoins(hash, coins) || mempool.isSpent(COutPoint(hash, n))) // TODO: this should be done by the CCoinsViewMemPool return NullUniValue; - mempool.pruneSpent(hash, coins); // TODO: this should be done by the CCoinsViewMemPool } else { if (!pcoinsTip->GetCoins(hash, coins)) return NullUniValue; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index a97b8805d68..3e310058b85 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -363,17 +363,10 @@ CTxMemPool::~CTxMemPool() delete minerPolicyEstimator; } -void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); - - auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); - - // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first->hash == hashTx) { - coins.Spend(it->first->n); // and remove those outputs from coins - it++; - } + return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const diff --git a/src/txmempool.h b/src/txmempool.h index 850d8382ee7..822e5619f52 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -557,7 +557,7 @@ class CTxMemPool void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector& vtxid); - void pruneSpent(const uint256& hash, CCoins &coins); + bool isSpent(const COutPoint& outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** From 546c1a1cbab1a22813dd1302ec2eaca72fe27823 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 26 Apr 2017 16:39:58 -0700 Subject: [PATCH 20/32] Refactor GetUTXOStats in preparation for per-COutPoint iteration (cherry picked from commit 4ec0d9e794e3f338e1ebb8b644ae890d2c2da2ee) --- src/rpc/blockchain.cpp | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7b933bbb551..a65b91c1ff3 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -864,13 +864,28 @@ struct CCoinsStats uint64_t nTransactions; uint64_t nTransactionOutputs; uint256 hashSerialized; - arith_uint256 nTotalAmount; uint64_t nDiskSize; arith_uint256 nTotalAmount; CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nDiskSize(0), nTotalAmount(0) {} }; +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << VARINT(output.second.out.nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { @@ -883,33 +898,25 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - arith_uint256 nTotalAmount = 0; while (pcursor->Valid()) { boost::this_thread::interruption_point(); uint256 key; CCoins coins; if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - ss << VARINT(coins.nHeight * 2 + coins.fCoinBase); + std::map outputs; for (unsigned int i=0; iNext(); } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; stats.nDiskSize = view->EstimateSize(); return true; } From 20dddd7487e2ab504c3b5cc29ab6e3bf1879dfbd Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 25 Apr 2017 11:29:39 -0700 Subject: [PATCH 21/32] Switch CCoinsView and chainstate db from per-txid to per-txout This patch makes several related changes: * Changes the CCoinsView virtual methods (GetCoins, HaveCoins, ...) to be COutPoint/Coin-based rather than txid/CCoins-based. * Changes the chainstate db to a new incompatible format that is also COutPoint/Coin based. * Implements reconstruction code for hash_serialized_2. * Adapts the coins_tests unit tests (thanks to Russell Yanofsky). A side effect of the new CCoinsView model is that we can no longer use the (unreliable) test for transaction outputs in the UTXO set to determine whether we already have a particular transaction. (cherry picked from commit 50830796889ecaa458871f1db878c255dd2554cb) --- src/coins.cpp | 130 ++--- src/coins.h | 83 ++-- src/init.cpp | 4 +- src/net_processing.cpp | 5 +- src/qt/transactiondesc.cpp | 7 +- src/rest.cpp | 111 +---- src/rpc/blockchain.cpp | 849 ++++++++++---------------------- src/rpc/rawtransaction.cpp | 7 +- src/test/coins_tests.cpp | 284 +++++------ src/test/test_bitcoin_fuzzy.cpp | 4 +- src/txdb.cpp | 72 ++- src/txdb.h | 16 +- src/txmempool.cpp | 190 ++----- src/txmempool.h | 160 ++---- src/validation.cpp | 54 +- 15 files changed, 684 insertions(+), 1292 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index ba56dbfd5ef..90949087508 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -42,41 +42,40 @@ bool CCoins::Spend(uint32_t nPos) return true; } -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } +bool CCoinsView::GetCoins(const COutPoint &outpoint, Coin &coin) const { return false; } +bool CCoinsView::HaveCoins(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } +bool CCoinsViewBacked::GetCoins(const COutPoint &outpoint, Coin &coin) const { return base->GetCoins(outpoint, coin); } +bool CCoinsViewBacked::HaveCoins(const COutPoint &outpoint) const { return base->HaveCoins(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) { } +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator CCoinsViewCache::FetchCoins(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; - CCoins tmp; - if (!base->GetCoins(txid, tmp)) + Coin tmp; + if (!base->GetCoins(outpoint, tmp)) return cacheCoins.end(); - CCoinsMap::iterator ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); + CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our + // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } @@ -84,10 +83,10 @@ CCoinsMap::iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::GetCoins(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoins(outpoint); if (it != cacheCoins.end()) { - coins = it->second.coins; + coin = it->second.coins; return true; } return false; @@ -98,23 +97,18 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi if (coin.out.scriptPubKey.IsUnspendable()) return; CCoinsMap::iterator it; bool inserted; - std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint.hash), std::tuple<>()); + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; if (!inserted) { cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); } if (!possible_overwrite) { - if (it->second.coins.IsAvailable(outpoint.n)) { + if (!it->second.coins.IsPruned()) { throw std::logic_error("Adding new coin that replaces non-pruned entry"); } - fresh = it->second.coins.IsPruned() && !(it->second.flags & CCoinsCacheEntry::DIRTY); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } - if (it->second.coins.vout.size() <= outpoint.n) { - it->second.coins.vout.resize(outpoint.n + 1); - } - it->second.coins.vout[outpoint.n] = std::move(coin.out); - it->second.coins.nHeight = coin.nHeight; - it->second.coins.fCoinBase = coin.fCoinBase; + it->second.coins = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); } @@ -130,58 +124,38 @@ void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { } void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { - CCoinsMap::iterator it = FetchCoins(outpoint.hash); + CCoinsMap::iterator it = FetchCoins(outpoint); if (it == cacheCoins.end()) return; cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); - if (moveout && it->second.coins.IsAvailable(outpoint.n)) { - *moveout = Coin(it->second.coins.vout[outpoint.n], it->second.coins.nHeight, it->second.coins.fCoinBase); + if (moveout) { + *moveout = std::move(it->second.coins); } - it->second.coins.Spend(outpoint.n); // Ignore return value: SpendCoin has no effect if no UTXO found. - if (it->second.coins.IsPruned() && it->second.flags & CCoinsCacheEntry::FRESH) { + if (it->second.flags & CCoinsCacheEntry::FRESH) { cacheCoins.erase(it); } else { - cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); it->second.flags |= CCoinsCacheEntry::DIRTY; - } -} - -const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - if (it == cacheCoins.end()) { - return NULL; - } else { - return &it->second.coins; + it->second.coins.Clear(); } } static const Coin coinEmpty; -const Coin CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { - CCoinsMap::const_iterator it = FetchCoins(outpoint.hash); - if (it == cacheCoins.end() || !it->second.coins.IsAvailable(outpoint.n)) { +const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoins(outpoint); + if (it == cacheCoins.end()) { return coinEmpty; } else { - return Coin(it->second.coins.vout[outpoint.n], it->second.coins.nHeight, it->second.coins.fCoinBase); + return it->second.coins; } } - -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); -} - bool CCoinsViewCache::HaveCoins(const COutPoint &outpoint) const { - CCoinsMap::const_iterator it = FetchCoins(outpoint.hash); - return (it != cacheCoins.end() && it->second.coins.IsAvailable(outpoint.n)); + CCoinsMap::const_iterator it = FetchCoins(outpoint); + return (it != cacheCoins.end() && !it->second.coins.IsPruned()); } -bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { - CCoinsMap::const_iterator it = cacheCoins.find(txid); +bool CCoinsViewCache::HaveCoinsInCache(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -206,7 +180,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); + entry.coins = std::move(it->second.coins); cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child @@ -233,7 +207,7 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } else { // A normal modification. cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); + itUs->second.coins = std::move(it->second.coins); cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in @@ -258,7 +232,7 @@ bool CCoinsViewCache::Flush() { return fOk; } -void CCoinsViewCache::Uncache(const uint256& hash) +void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { @@ -273,9 +247,9 @@ unsigned int CCoinsViewCache::GetCacheSize() const { const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const { - const CCoins* coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; + const Coin& coin = AccessCoin(input.prevout); + assert(!coin.IsPruned()); + return coin.out; } CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const @@ -294,9 +268,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + if (!HaveCoins(tx.vin[i].prevout)) { return false; } } @@ -304,31 +276,9 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } -double CCoinsViewCache::GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const -{ - inChainInputValue = 0; - if (tx.IsCoinBase()) - return 0.0; - double dResult = 0.0; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - { - const CCoins* coins = AccessCoins(txin.prevout.hash); - assert(coins); - if (!coins->IsAvailable(txin.prevout.n)) continue; - if (coins->nHeight <= nHeight) { - dResult += (double)(coins->vout[txin.prevout.n].nValue) * (nHeight-coins->nHeight); - inChainInputValue += coins->vout[txin.prevout.n].nValue; - } - } - return tx.ComputePriority(dResult); -} -CCoinsViewCursor::~CCoinsViewCursor() -{ -} - static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. -const Coin AccessByTxid(const CCoinsViewCache& view, const uint256& txid) +const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { COutPoint iter(txid, 0); while (iter.n < MAX_OUTPUTS_PER_BLOCK) { diff --git a/src/coins.h b/src/coins.h index 0a5def1697e..8fd63c3fb2f 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" @@ -298,28 +299,28 @@ class CCoins } }; -class SaltedTxidHasher +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint& id) const { + return SipHashUint256Extra(k0, k1, id.hash, id.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + Coin coins; // The actual cached data. unsigned char flags; enum Flags { @@ -332,21 +333,21 @@ struct CCoinsCacheEntry */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coins(std::move(coin_)), flags(0) {} }; -typedef std::unordered_map CCoinsMap; +typedef std::unordered_map CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; - /* Don't care about GetKeySize here */ + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -362,17 +363,17 @@ class CCoinsViewCursor class CCoinsView { public: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; + //! Retrieve the Coin (unspent transaction output) for a given outpoint. + virtual bool GetCoins(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; + //! Just check whether we have data for a given outpoint. + //! This may (but cannot always) return true for spent outputs. + virtual bool HaveCoins(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -395,12 +396,12 @@ class CCoinsViewBacked : public CCoinsView public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; + bool GetCoins(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoins(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; size_t EstimateSize() const override; }; @@ -416,41 +417,32 @@ class CCoinsViewCache : public CCoinsViewBacked mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); // Standard CCoinsView methods - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoins(const COutPoint &outpoint, Coin &coin) const; bool HaveCoins(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); /** - * Check if we have the given tx already loaded in this cache. + * Check if we have the given utxo already loaded in this cache. * The semantics are the same as HaveCoins(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const uint256 &txid) const; - - /** - * Return a pointer to CCoins in the cache, or NULL if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are - * allowed while accessing the returned pointer. - */ - const CCoins* AccessCoins(const uint256 &txid) const; + bool HaveCoinsInCache(const COutPoint &outpoint) const; /** - * Return a copy of a Coin in the cache, or a pruned one if not found. This is + * Return a reference to Coin in the cache, or a pruned one if not found. This is * more efficient than GetCoins. Modifications to other cache entries are * allowed while accessing the returned pointer. - * TODO: return a reference to a Coin after changing CCoinsViewCache storage. */ - const Coin AccessCoin(const COutPoint &output) const; + const Coin& AccessCoin(const COutPoint &output) const; /** * Add a coin. Set potential_overwrite to true if a non-pruned version may @@ -473,12 +465,12 @@ class CCoinsViewCache : public CCoinsViewBacked bool Flush(); /** - * Removes the transaction with the given hash from the cache, if it is + * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ - void Uncache(const uint256 &txid); + void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -497,17 +489,10 @@ class CCoinsViewCache : public CCoinsViewBacked //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - /** - * Return priority of tx at height nHeight. Also calculate the sum of the values of the inputs - * that are already in the chain. These are the inputs that will age and increase priority as - * new blocks are added to the chain. - */ - double GetPriority(const CTransaction &tx, int nHeight, CAmount &inChainInputValue) const; - const CTxOut &GetOutputFor(const CTxIn& input) const; private: - CCoinsMap::iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoins(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -522,6 +507,6 @@ class CCoinsViewCache : public CCoinsViewBacked void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); //! Utility function to find any unspent output with a given txid. -const Coin AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); +const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); #endif // BITCOIN_COINS_H diff --git a/src/init.cpp b/src/init.cpp index 4e1b8964acc..1e43b36f7bb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -152,9 +152,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoins(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoins(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 443d5266047..bd514dce8e2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -987,12 +987,11 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); + pcoinsTip->HaveCoinsInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinsInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index aadd5bf4df0..84cdc65b6fb 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -790,13 +790,12 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco { COutPoint prevout = txin.prevout; - CCoins prev; - if(pcoinsTip->GetCoins(prevout.hash, prev)) + Coin prev; + if(pcoinsTip->GetCoins(prevout, prev)) { - if (prevout.n < prev.vout.size()) { strHTML += "
  • "; - const CTxOut &vout = prev.vout[prevout.n]; + const CTxOut &vout = prev.out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/rest.cpp b/src/rest.cpp index 4bafe6bb7d3..16c8c4f5293 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,15 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2022 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "chain.h" #include "chainparams.h" +#include "core_io.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "validation.h" #include "httpserver.h" +#include "rpc/blockchain.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" @@ -46,6 +47,9 @@ struct CCoin { ADD_SERIALIZE_METHODS; + CCoin() : nHeight(0) {} + CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} + template inline void SerializationOp(Stream& s, Operation ser_action) { @@ -56,13 +60,6 @@ struct CCoin { } }; -extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); -extern UniValue mempoolInfoToJSON(); -extern UniValue mempoolToJSON(bool fVerbose = false); -extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); -extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); - static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); @@ -161,9 +158,8 @@ static bool rest_headers(HTTPRequest* req, } CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); - const CChainParams& chainparams = Params(); BOOST_FOREACH(const CBlockIndex *pindex, headers) { - ssHeader << pindex->GetBlockHeader(chainparams.GetConsensus(pindex->nHeight)); + ssHeader << pindex->GetBlockHeader(); } switch (rf) { @@ -223,7 +219,7 @@ static bool rest_block(HTTPRequest* req, if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus(pblockindex->nHeight))) + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -364,7 +360,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) CTransactionRef tx; uint256 hashBlock = uint256(); - if (!GetTransaction(hash, tx, Params().GetConsensus(0), hashBlock, true)) + if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); @@ -387,7 +383,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, objTx); + TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -462,7 +458,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) std::vector strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); } - // Falls through case RF_BINARY: { try { @@ -513,27 +508,15 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) CCoinsViewCache& viewChain = *pcoinsTip; CCoinsViewMemPool viewMempool(&viewChain, mempool); - if (fCheckMemPool) { - view.SetBackend(viewMempool); // set cache backend to db+mempool in case user likes to query mempool - } else { - view.SetBackend(viewChain); // set cache backend to db only otherwise - } + if (fCheckMemPool) + view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool for (size_t i = 0; i < vOutPoints.size(); i++) { - CCoins coins; - uint256 hash = vOutPoints[i].hash; bool hit = false; - if (view.GetCoins(hash, coins)) { - if (coins.IsAvailable(vOutPoints[i].n) && (!fCheckMemPool || !mempool.isSpent(vOutPoints[i]))) { - hit = true; - // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if - // n is valid but points to an already spent output (IsNull). - CCoin coin; - coin.nHeight = coins.nHeight; - coin.out = coins.vout.at(vOutPoints[i].n); - assert(!coin.out.IsNull()); - outs.push_back(coin); - } + Coin coin; + if (view.GetCoins(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + hit = true; + outs.emplace_back(std::move(coin)); } hits.push_back(hit); @@ -570,23 +553,23 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) // pack in some essentials // use more or less the same output as mentioned in Bip64 - objGetUTXOResponse.pushKV("chainHeight", chainActive.Height()); - objGetUTXOResponse.pushKV("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex()); - objGetUTXOResponse.pushKV("bitmap", bitmapStringRepresentation); + objGetUTXOResponse.push_back(Pair("chainHeight", chainActive.Height())); + objGetUTXOResponse.push_back(Pair("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex())); + objGetUTXOResponse.push_back(Pair("bitmap", bitmapStringRepresentation)); UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.pushKV("height", (int32_t)coin.nHeight); - utxo.pushKV("value", ValueFromAmount(coin.out.nValue)); + utxo.push_back(Pair("height", (int32_t)coin.nHeight)); + utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); - utxo.pushKV("scriptPubKey", o); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); + utxo.push_back(Pair("scriptPubKey", o)); utxos.push_back(utxo); } - objGetUTXOResponse.pushKV("utxos", utxos); + objGetUTXOResponse.push_back(Pair("utxos", utxos)); // return json string std::string strJSON = objGetUTXOResponse.write() + "\n"; @@ -603,53 +586,6 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) return true; // continue to process further HTTP reqs on this cxn } -static bool rest_blockhash_by_height(HTTPRequest* req, - const std::string& str_uri_part) - { - if (!CheckWarmup(req)) return false; - std::string height_str; - const RetFormat rf = ParseDataFormat(height_str, str_uri_part); - - int32_t blockheight; - if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { - return RESTERR(req, HTTP_BAD_REQUEST, "Invalid height: " + SanitizeString(height_str)); - } - - CBlockIndex* pblockindex = nullptr; - { - LOCK(cs_main); - if (blockheight > chainActive.Height()) { - return RESTERR(req, HTTP_NOT_FOUND, "Block height out of range"); - } - pblockindex = chainActive[blockheight]; - } - switch (rf) { - case RF_BINARY: { - CDataStream ss_blockhash(SER_NETWORK, PROTOCOL_VERSION); - ss_blockhash << pblockindex->GetBlockHash(); - req->WriteHeader("Content-Type", "application/octet-stream"); - req->WriteReply(HTTP_OK, ss_blockhash.str()); - return true; - } - case RF_HEX: { - req->WriteHeader("Content-Type", "text/plain"); - req->WriteReply(HTTP_OK, pblockindex->GetBlockHash().GetHex() + "\n"); - return true; - } - case RF_JSON: { - req->WriteHeader("Content-Type", "application/json"); - UniValue resp = UniValue(UniValue::VOBJ); - resp.pushKV("blockhash", pblockindex->GetBlockHash().GetHex()); - req->WriteReply(HTTP_OK, resp.write() + "\n"); - return true; - } - default: { - return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); - } - } - } - - static const struct { const char* prefix; bool (*handler)(HTTPRequest* req, const std::string& strReq); @@ -662,7 +598,6 @@ static const struct { {"/rest/mempool/contents", rest_mempool_contents}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, - {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; bool StartREST() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a65b91c1ff3..77f1dcb21f7 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,6 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2022-2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,16 +11,15 @@ #include "checkpoints.h" #include "coins.h" #include "consensus/validation.h" -#include "core_io.h" -#include "dogecoin.h" #include "validation.h" +#include "core_io.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" #include "txmempool.h" -#include "undo.h" #include "util.h" #include "utilstrencodings.h" #include "hash.h" @@ -30,12 +28,10 @@ #include -#include #include // boost::thread::interrupt #include #include -using namespace std; struct CUpdatedBlock { @@ -48,15 +44,12 @@ static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); double GetDifficulty(const CBlockIndex* blockindex) { - // Floating point number that is a multiple of the minimum difficulty, - // minimum difficulty = 1.0. - if (blockindex == nullptr) + if (blockindex == NULL) { - if (chainActive.Tip() == nullptr) + if (chainActive.Tip() == NULL) return 1.0; else blockindex = chainActive.Tip(); @@ -81,121 +74,82 @@ double GetDifficulty(const CBlockIndex* blockindex) return dDiff; } -UniValue AuxpowToJSON(const CAuxPow& auxpow) -{ - UniValue result(UniValue::VOBJ); - - { - UniValue tx(UniValue::VOBJ); - tx.pushKV("hex", EncodeHexTx(auxpow)); - TxToJSON(auxpow, auxpow.parentBlock.GetHash(), tx); - result.pushKV("tx", tx); - } - - result.pushKV("index", auxpow.nIndex); - result.pushKV("chainindex", auxpow.nChainIndex); - - { - UniValue branch(UniValue::VARR); - BOOST_FOREACH(const uint256& node, auxpow.vMerkleBranch) - branch.push_back(node.GetHex()); - result.pushKV("merklebranch", branch); - } - - { - UniValue branch(UniValue::VARR); - BOOST_FOREACH(const uint256& node, auxpow.vChainMerkleBranch) - branch.push_back(node.GetHex()); - result.pushKV("chainmerklebranch", branch); - } - - CDataStream ssParent(SER_NETWORK, PROTOCOL_VERSION); - ssParent << auxpow.parentBlock; - const std::string strHex = HexStr(ssParent.begin(), ssParent.end()); - result.pushKV("parentblock", strHex); - - return result; -} - UniValue blockheaderToJSON(const CBlockIndex* blockindex) { UniValue result(UniValue::VOBJ); - result.pushKV("hash", blockindex->GetBlockHash().GetHex()); + result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); int confirmations = -1; // Only report confirmations if the block is on the main chain if (chainActive.Contains(blockindex)) confirmations = chainActive.Height() - blockindex->nHeight + 1; - result.pushKV("confirmations", confirmations); - result.pushKV("height", blockindex->nHeight); - result.pushKV("version", blockindex->nVersion); - result.pushKV("versionHex", strprintf("%08x", blockindex->nVersion)); - result.pushKV("merkleroot", blockindex->hashMerkleRoot.GetHex()); - result.pushKV("time", (int64_t)blockindex->nTime); - result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); - result.pushKV("nonce", (uint64_t)blockindex->nNonce); - result.pushKV("bits", strprintf("%08x", blockindex->nBits)); - result.pushKV("difficulty", GetDifficulty(blockindex)); - result.pushKV("chainwork", blockindex->nChainWork.GetHex()); + result.push_back(Pair("confirmations", confirmations)); + result.push_back(Pair("height", blockindex->nHeight)); + result.push_back(Pair("version", blockindex->nVersion)); + result.push_back(Pair("versionHex", strprintf("%08x", blockindex->nVersion))); + result.push_back(Pair("merkleroot", blockindex->hashMerkleRoot.GetHex())); + result.push_back(Pair("time", (int64_t)blockindex->nTime)); + result.push_back(Pair("mediantime", (int64_t)blockindex->GetMedianTimePast())); + result.push_back(Pair("nonce", (uint64_t)blockindex->nNonce)); + result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits))); + result.push_back(Pair("difficulty", GetDifficulty(blockindex))); + result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex())); if (blockindex->pprev) - result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); + result.push_back(Pair("previousblockhash", blockindex->pprev->GetBlockHash().GetHex())); CBlockIndex *pnext = chainActive.Next(blockindex); if (pnext) - result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); + result.push_back(Pair("nextblockhash", pnext->GetBlockHash().GetHex())); return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails) { UniValue result(UniValue::VOBJ); - result.pushKV("hash", blockindex->GetBlockHash().GetHex()); + result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); int confirmations = -1; // Only report confirmations if the block is on the main chain if (chainActive.Contains(blockindex)) confirmations = chainActive.Height() - blockindex->nHeight + 1; - result.pushKV("confirmations", confirmations); - result.pushKV("strippedsize", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); - result.pushKV("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION)); - result.pushKV("weight", (int)::GetBlockWeight(block)); - result.pushKV("height", blockindex->nHeight); - result.pushKV("version", block.nVersion); - result.pushKV("versionHex", strprintf("%08x", block.nVersion)); - result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); + result.push_back(Pair("confirmations", confirmations)); + result.push_back(Pair("strippedsize", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS))); + result.push_back(Pair("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION))); + result.push_back(Pair("weight", (int)::GetBlockWeight(block))); + result.push_back(Pair("height", blockindex->nHeight)); + result.push_back(Pair("version", block.nVersion)); + result.push_back(Pair("versionHex", strprintf("%08x", block.nVersion))); + result.push_back(Pair("merkleroot", block.hashMerkleRoot.GetHex())); UniValue txs(UniValue::VARR); for(const auto& tx : block.vtx) { if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, uint256(), objTx); + TxToUniv(*tx, uint256(), objTx); txs.push_back(objTx); } else txs.push_back(tx->GetHash().GetHex()); } - result.pushKV("tx", txs); - result.pushKV("time", block.GetBlockTime()); - result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); - result.pushKV("nonce", (uint64_t)block.nNonce); - result.pushKV("bits", strprintf("%08x", block.nBits)); - result.pushKV("difficulty", GetDifficulty(blockindex)); - result.pushKV("chainwork", blockindex->nChainWork.GetHex()); - - if (block.auxpow) - result.pushKV("auxpow", AuxpowToJSON(*block.auxpow)); + result.push_back(Pair("tx", txs)); + result.push_back(Pair("time", block.GetBlockTime())); + result.push_back(Pair("mediantime", (int64_t)blockindex->GetMedianTimePast())); + result.push_back(Pair("nonce", (uint64_t)block.nNonce)); + result.push_back(Pair("bits", strprintf("%08x", block.nBits))); + result.push_back(Pair("difficulty", GetDifficulty(blockindex))); + result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex())); if (blockindex->pprev) - result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); + result.push_back(Pair("previousblockhash", blockindex->pprev->GetBlockHash().GetHex())); CBlockIndex *pnext = chainActive.Next(blockindex); if (pnext) - result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); + result.push_back(Pair("nextblockhash", pnext->GetBlockHash().GetHex())); return result; } UniValue getblockcount(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getblockcount\n" "\nReturns the number of blocks in the longest blockchain.\n" "\nResult:\n" @@ -212,7 +166,7 @@ UniValue getblockcount(const JSONRPCRequest& request) UniValue getbestblockhash(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getbestblockhash\n" "\nReturns the hash of the best (tip) block in the longest blockchain.\n" "\nResult:\n" @@ -233,13 +187,13 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } - cond_blockchange.notify_all(); + cond_blockchange.notify_all(); } UniValue waitfornewblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "waitfornewblock (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" @@ -269,15 +223,15 @@ UniValue waitfornewblock(const JSONRPCRequest& request) block = latestblock; } UniValue ret(UniValue::VOBJ); - ret.pushKV("hash", block.hash.GetHex()); - ret.pushKV("height", block.height); + ret.push_back(Pair("hash", block.hash.GetHex())); + ret.push_back(Pair("height", block.height)); return ret; } UniValue waitforblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "waitforblock (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" @@ -311,15 +265,15 @@ UniValue waitforblock(const JSONRPCRequest& request) } UniValue ret(UniValue::VOBJ); - ret.pushKV("hash", block.hash.GetHex()); - ret.pushKV("height", block.height); + ret.push_back(Pair("hash", block.hash.GetHex())); + ret.push_back(Pair("height", block.height)); return ret; } UniValue waitforblockheight(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "waitforblockheight (timeout)\n" "\nWaits for (at least) block height and returns the height and hash\n" "of the current tip.\n" @@ -353,15 +307,15 @@ UniValue waitforblockheight(const JSONRPCRequest& request) block = latestblock; } UniValue ret(UniValue::VOBJ); - ret.pushKV("hash", block.hash.GetHex()); - ret.pushKV("height", block.height); + ret.push_back(Pair("hash", block.hash.GetHex())); + ret.push_back(Pair("height", block.height)); return ret; } UniValue getdifficulty(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getdifficulty\n" "\nReturns the proof-of-work difficulty as a multiple of the minimum difficulty.\n" "\nResult:\n" @@ -382,8 +336,6 @@ std::string EntryDescriptionString() " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" " \"height\" : n, (numeric) block height when transaction entered pool\n" - " \"startingpriority\" : n, (numeric) DEPRECATED. Priority when transaction entered pool\n" - " \"currentpriority\" : n, (numeric) DEPRECATED. Transaction priority now\n" " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n" " \"descendantsize\" : n, (numeric) virtual transaction size of in-mempool descendants (including this one)\n" " \"descendantfees\" : n, (numeric) modified fees (see above) of in-mempool descendants (including this one)\n" @@ -399,21 +351,19 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) { AssertLockHeld(mempool.cs); - info.pushKV("size", (int)e.GetTxSize()); - info.pushKV("fee", ValueFromAmount(e.GetFee())); - info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); - info.pushKV("time", e.GetTime()); - info.pushKV("height", (int)e.GetHeight()); - info.pushKV("startingpriority", e.GetPriority(e.GetHeight())); - info.pushKV("currentpriority", e.GetPriority(chainActive.Height())); - info.pushKV("descendantcount", e.GetCountWithDescendants()); - info.pushKV("descendantsize", e.GetSizeWithDescendants()); - info.pushKV("descendantfees", e.GetModFeesWithDescendants()); - info.pushKV("ancestorcount", e.GetCountWithAncestors()); - info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); + info.push_back(Pair("size", (int)e.GetTxSize())); + info.push_back(Pair("fee", ValueFromAmount(e.GetFee()))); + info.push_back(Pair("modifiedfee", ValueFromAmount(e.GetModifiedFee()))); + info.push_back(Pair("time", e.GetTime())); + info.push_back(Pair("height", (int)e.GetHeight())); + info.push_back(Pair("descendantcount", e.GetCountWithDescendants())); + info.push_back(Pair("descendantsize", e.GetSizeWithDescendants())); + info.push_back(Pair("descendantfees", e.GetModFeesWithDescendants())); + info.push_back(Pair("ancestorcount", e.GetCountWithAncestors())); + info.push_back(Pair("ancestorsize", e.GetSizeWithAncestors())); + info.push_back(Pair("ancestorfees", e.GetModFeesWithAncestors())); const CTransaction& tx = e.GetTx(); - set setDepends; + std::set setDepends; BOOST_FOREACH(const CTxIn& txin, tx.vin) { if (mempool.exists(txin.prevout.hash)) @@ -421,15 +371,15 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) } UniValue depends(UniValue::VARR); - BOOST_FOREACH(const string& dep, setDepends) + BOOST_FOREACH(const std::string& dep, setDepends) { depends.push_back(dep); } - info.pushKV("depends", depends); + info.push_back(Pair("depends", depends)); } -UniValue mempoolToJSON(bool fVerbose = false) +UniValue mempoolToJSON(bool fVerbose) { if (fVerbose) { @@ -440,13 +390,13 @@ UniValue mempoolToJSON(bool fVerbose = false) const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(info, e); - o.pushKV(hash.ToString(), info); + o.push_back(Pair(hash.ToString(), info)); } return o; } else { - vector vtxid; + std::vector vtxid; mempool.queryHashes(vtxid); UniValue a(UniValue::VARR); @@ -460,7 +410,7 @@ UniValue mempoolToJSON(bool fVerbose = false) UniValue getrawmempool(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw runtime_error( + throw std::runtime_error( "getrawmempool ( verbose )\n" "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" @@ -492,7 +442,7 @@ UniValue getrawmempool(const JSONRPCRequest& request) UniValue getmempoolancestors(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw runtime_error( + throw std::runtime_error( "getmempoolancestors txid (verbose)\n" "\nIf txid is in the mempool, returns all in-mempool ancestors.\n" "\nArguments:\n" @@ -547,7 +497,7 @@ UniValue getmempoolancestors(const JSONRPCRequest& request) const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(info, e); - o.pushKV(_hash.ToString(), info); + o.push_back(Pair(_hash.ToString(), info)); } return o; } @@ -556,7 +506,7 @@ UniValue getmempoolancestors(const JSONRPCRequest& request) UniValue getmempooldescendants(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw runtime_error( + throw std::runtime_error( "getmempooldescendants txid (verbose)\n" "\nIf txid is in the mempool, returns all in-mempool descendants.\n" "\nArguments:\n" @@ -611,7 +561,7 @@ UniValue getmempooldescendants(const JSONRPCRequest& request) const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(info, e); - o.pushKV(_hash.ToString(), info); + o.push_back(Pair(_hash.ToString(), info)); } return o; } @@ -620,7 +570,7 @@ UniValue getmempooldescendants(const JSONRPCRequest& request) UniValue getmempoolentry(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) { - throw runtime_error( + throw std::runtime_error( "getmempoolentry txid\n" "\nReturns mempool data for given transaction\n" "\nArguments:\n" @@ -653,7 +603,7 @@ UniValue getmempoolentry(const JSONRPCRequest& request) UniValue getblockhash(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "getblockhash height\n" "\nReturns hash of block in best-block-chain at height provided.\n" "\nArguments:\n" @@ -678,7 +628,7 @@ UniValue getblockhash(const JSONRPCRequest& request) UniValue getblockheader(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getblockheader \"hash\" ( verbose )\n" "\nIf verbose is false, returns a string that is serialized, hex-encoded data for blockheader 'hash'.\n" "If verbose is true, returns an Object with information about blockheader .\n" @@ -726,7 +676,7 @@ UniValue getblockheader(const JSONRPCRequest& request) if (!fVerbose) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); - ssBlock << pblockindex->GetBlockHeader(Params().GetConsensus(pblockindex->nHeight)); + ssBlock << pblockindex->GetBlockHeader(); std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); return strHex; } @@ -734,47 +684,10 @@ UniValue getblockheader(const JSONRPCRequest& request) return blockheaderToJSON(pblockindex); } -static CBlock GetBlockChecked(const CBlockIndex* pblockindex) -{ - CBlock block; - if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - } - - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus(pblockindex->nHeight))) { - // Block not found on disk. This could be because we have the block - // header in our index but don't have the block (for example if a - // non-whitelisted node sends us an unrequested long chain of valid - // blocks, we add the headers to our index, but don't accept the - // block). - throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); - } - - return block; -} - -static CBlockUndo GetUndoChecked(const CBlockIndex* pblockindex) -{ - CBlockUndo blockUndo; - - // The Genesis block does not have undo data - if (pblockindex->nHeight == 0) return blockUndo; - - if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { - throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); - } - - if (!UndoReadFromDisk(blockUndo, pblockindex->GetUndoPos(), pblockindex->pprev->GetBlockHash())) { - throw JSONRPCError(RPC_MISC_ERROR, "Can't read undo data from disk"); - } - - return blockUndo; -} - UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "getblock \"blockhash\" ( verbosity ) \n" "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block .\n" @@ -821,30 +734,35 @@ UniValue getblock(const JSONRPCRequest& request) + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") ); + LOCK(cs_main); + std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int verbosity = 1; if (request.params.size() > 1) { - if(request.params[1].isNum()) { + if(request.params[1].isNum()) verbosity = request.params[1].get_int(); - } else { + else verbosity = request.params[1].get_bool() ? 1 : 0; - } } - CBlock block; - const CBlockIndex* pblockindex; - { - LOCK(cs_main); + if (mapBlockIndex.count(hash) == 0) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - if (mapBlockIndex.count(hash) == 0) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + CBlock block; + CBlockIndex* pblockindex = mapBlockIndex[hash]; - pblockindex = mapBlockIndex[hash]; + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); - block = GetBlockChecked(pblockindex); - } + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) + // Block not found on disk. This could be because we have the block + // header in our index but don't have the block (for example if a + // non-whitelisted node sends us an unrequested long chain of valid + // blocks, we add the headers to our index, but don't accept the + // block). + throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); if (verbosity <= 0) { @@ -865,9 +783,9 @@ struct CCoinsStats uint64_t nTransactionOutputs; uint256 hashSerialized; uint64_t nDiskSize; - arith_uint256 nTotalAmount; + CAmount nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nDiskSize(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nTotalAmount(0) {} }; static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) @@ -898,24 +816,27 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; + uint256 prevkey; + std::map outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - std::map outputs; - for (unsigned int i=0; iGetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - ApplyStats(stats, ss, key, outputs); + prevkey = key.hash; + outputs[key.n] = std::move(coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); stats.nDiskSize = view->EstimateSize(); return true; @@ -924,7 +845,7 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "pruneblockchain\n" "\nArguments:\n" "1. \"height\" (numeric, required) The block height to prune up to. May be set to a discrete height, or a unix timestamp\n" @@ -948,7 +869,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - 7200); + CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -962,7 +883,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) else if (height > chainHeight) throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height."); else if (height > chainHeight - MIN_BLOCKS_TO_KEEP) { - LogPrint("rpc", "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); + LogPrint(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); height = chainHeight - MIN_BLOCKS_TO_KEEP; } @@ -973,7 +894,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) UniValue gettxoutsetinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "gettxoutsetinfo\n" "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n" @@ -983,7 +904,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" + " \"hash_serialized\": \"hash\", (string) The serialized hash\n" " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" @@ -997,13 +918,13 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); if (GetUTXOStats(pcoinsTip, stats)) { - ret.pushKV("height", (int64_t)stats.nHeight); - ret.pushKV("bestblock", stats.hashBlock.GetHex()); - ret.pushKV("transactions", (int64_t)stats.nTransactions); - ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); - ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); - ret.pushKV("disk_size", (int64_t)stats.nDiskSize); - ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); + ret.push_back(Pair("height", (int64_t)stats.nHeight)); + ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); + ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); + ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); + ret.push_back(Pair("hash_serialized_2", stats.hashSerialized.GetHex())); + ret.push_back(Pair("disk_size", stats.nDiskSize)); + ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -1013,7 +934,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) UniValue gettxout(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) - throw runtime_error( + throw std::runtime_error( "gettxout \"txid\" n ( include_mempool )\n" "\nReturns details about an unspent transaction output.\n" "\nArguments:\n" @@ -1030,11 +951,12 @@ UniValue gettxout(const JSONRPCRequest& request) " \"hex\" : \"hex\", (string) \n" " \"reqSigs\" : n, (numeric) Number of required signatures\n" " \"type\" : \"pubkeyhash\", (string) The type, eg pubkeyhash\n" - " \"addresses\" : [ (array of string) array of dogecoin addresses\n" - " \"address\" (string) dogecoin address\n" + " \"addresses\" : [ (array of string) array of bitcoin addresses\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ]\n" " },\n" + " \"version\" : n, (numeric) The version\n" " \"coinbase\" : true|false (boolean) Coinbase or not\n" "}\n" @@ -1054,35 +976,37 @@ UniValue gettxout(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int n = request.params[1].get_int(); + COutPoint out(hash, n); bool fMempool = true; if (request.params.size() > 2) fMempool = request.params[2].get_bool(); - CCoins coins; + Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins) || mempool.isSpent(COutPoint(hash, n))) // TODO: this should be done by the CCoinsViewMemPool + if (!view.GetCoins(out, coin) || mempool.isSpent(out)) { // TODO: filtering spent coins should be done by the CCoinsViewMemPool return NullUniValue; + } } else { - if (!pcoinsTip->GetCoins(hash, coins)) + if (!pcoinsTip->GetCoins(out, coin)) { return NullUniValue; + } } - if (n<0 || (unsigned int)n>=coins.vout.size() || coins.vout[n].IsNull()) - return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; - ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); - if ((unsigned int)coins.nHeight == MEMPOOL_HEIGHT) - ret.pushKV("confirmations", 0); - else - ret.pushKV("confirmations", pindex->nHeight - coins.nHeight + 1); - ret.pushKV("value", ValueFromAmount(coins.vout[n].nValue)); + ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); + if (coin.nHeight == MEMPOOL_HEIGHT) { + ret.push_back(Pair("confirmations", 0)); + } else { + ret.push_back(Pair("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1))); + } + ret.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); - ret.pushKV("scriptPubKey", o); - ret.pushKV("coinbase", coins.fCoinBase); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); + ret.push_back(Pair("scriptPubKey", o)); + ret.push_back(Pair("coinbase", (bool)coin.fCoinBase)); return ret; } @@ -1092,7 +1016,7 @@ UniValue verifychain(const JSONRPCRequest& request) int nCheckLevel = GetArg("-checklevel", DEFAULT_CHECKLEVEL); int nCheckDepth = GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); if (request.fHelp || request.params.size() > 2) - throw runtime_error( + throw std::runtime_error( "verifychain ( checklevel nblocks )\n" "\nVerifies blockchain database.\n" "\nArguments:\n" @@ -1109,13 +1033,8 @@ UniValue verifychain(const JSONRPCRequest& request) if (request.params.size() > 0) nCheckLevel = request.params[0].get_int(); - if (nCheckLevel < 0 || nCheckLevel > 4) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: checklevel must be >= 0 and <= 4"); - if (request.params.size() > 1) nCheckDepth = request.params[1].get_int(); - if (nCheckDepth < 0) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: nblocks must be >= 0"); return CVerifyDB().VerifyDB(Params(), pcoinsTip, nCheckLevel, nCheckDepth); } @@ -1137,16 +1056,16 @@ static UniValue SoftForkMajorityDesc(int version, CBlockIndex* pindex, const Con activated = pindex->nHeight >= consensusParams.BIP65Height; break; } - rv.pushKV("status", activated); + rv.push_back(Pair("status", activated)); return rv; } static UniValue SoftForkDesc(const std::string &name, int version, CBlockIndex* pindex, const Consensus::Params& consensusParams) { UniValue rv(UniValue::VOBJ); - rv.pushKV("id", name); - rv.pushKV("version", version); - rv.pushKV("reject", SoftForkMajorityDesc(version, pindex, consensusParams)); + rv.push_back(Pair("id", name)); + rv.push_back(Pair("version", version)); + rv.push_back(Pair("reject", SoftForkMajorityDesc(version, pindex, consensusParams))); return rv; } @@ -1155,19 +1074,30 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse UniValue rv(UniValue::VOBJ); const ThresholdState thresholdState = VersionBitsTipState(consensusParams, id); switch (thresholdState) { - case THRESHOLD_DEFINED: rv.pushKV("status", "defined"); break; - case THRESHOLD_STARTED: rv.pushKV("status", "started"); break; - case THRESHOLD_LOCKED_IN: rv.pushKV("status", "locked_in"); break; - case THRESHOLD_ACTIVE: rv.pushKV("status", "active"); break; - case THRESHOLD_FAILED: rv.pushKV("status", "failed"); break; + case THRESHOLD_DEFINED: rv.push_back(Pair("status", "defined")); break; + case THRESHOLD_STARTED: rv.push_back(Pair("status", "started")); break; + case THRESHOLD_LOCKED_IN: rv.push_back(Pair("status", "locked_in")); break; + case THRESHOLD_ACTIVE: rv.push_back(Pair("status", "active")); break; + case THRESHOLD_FAILED: rv.push_back(Pair("status", "failed")); break; } if (THRESHOLD_STARTED == thresholdState) { - rv.pushKV("bit", consensusParams.vDeployments[id].bit); + rv.push_back(Pair("bit", consensusParams.vDeployments[id].bit)); + } + rv.push_back(Pair("startTime", consensusParams.vDeployments[id].nStartTime)); + rv.push_back(Pair("timeout", consensusParams.vDeployments[id].nTimeout)); + rv.push_back(Pair("since", VersionBitsTipStateSinceHeight(consensusParams, id))); + if (THRESHOLD_STARTED == thresholdState) + { + UniValue statsUV(UniValue::VOBJ); + BIP9Stats statsStruct = VersionBitsTipStatistics(consensusParams, id); + statsUV.push_back(Pair("period", statsStruct.period)); + statsUV.push_back(Pair("threshold", statsStruct.threshold)); + statsUV.push_back(Pair("elapsed", statsStruct.elapsed)); + statsUV.push_back(Pair("count", statsStruct.count)); + statsUV.push_back(Pair("possible", statsStruct.possible)); + rv.push_back(Pair("statistics", statsUV)); } - rv.pushKV("startTime", consensusParams.vDeployments[id].nStartTime); - rv.pushKV("timeout", consensusParams.vDeployments[id].nTimeout); - rv.pushKV("since", VersionBitsTipStateSinceHeight(consensusParams, id)); return rv; } @@ -1177,13 +1107,13 @@ void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const std::string &name, // A timeout value of 0 guarantees a softfork will never be activated. // This is used when softfork codes are merged without specifying the deployment schedule. if (consensusParams.vDeployments[id].nTimeout > 0) - bip9_softforks.pushKV(name, BIP9SoftForkDesc(consensusParams, id)); + bip9_softforks.push_back(Pair(name, BIP9SoftForkDesc(consensusParams, id))); } UniValue getblockchaininfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getblockchaininfo\n" "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" @@ -1195,13 +1125,9 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" - " \"initialblockdownload\": xxxx, (bool) (debug information) estimate of whether this node is in Initial Block Download mode.\n" " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" - " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" - " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" - " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" - " \"prune_target_size\": xxxxxx, (numeric) the target size used by pruning (only present if automatic pruning is enabled)\n" + " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored\n" " \"softforks\": [ (array) status of softforks in progress\n" " {\n" " \"id\": \"xxxx\", (string) name of softfork\n" @@ -1217,7 +1143,14 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx (numeric) height of the first block to which the status applies\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " }\n" " }\n" " }\n" "}\n" @@ -1229,35 +1162,17 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) LOCK(cs_main); UniValue obj(UniValue::VOBJ); - obj.pushKV("chain", Params().NetworkIDString()); - obj.pushKV("blocks", (int)chainActive.Height()); - obj.pushKV("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1); - obj.pushKV("bestblockhash", chainActive.Tip()->GetBlockHash().GetHex()); - obj.pushKV("difficulty", (double)GetDifficulty()); - obj.pushKV("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast()); - obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip())); - obj.pushKV("initialblockdownload", IsInitialBlockDownload()); - obj.pushKV("chainwork", chainActive.Tip()->nChainWork.GetHex()); - obj.pushKV("size_on_disk", CalculateCurrentUsage()); - obj.pushKV("pruned", fPruneMode); - if (fPruneMode) { - CBlockIndex* block = chainActive.Tip(); - assert(block); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - - obj.pushKV("pruneheight", block->nHeight); - - // if 0, execution bypasses the whole if block. - bool automatic_pruning = (GetArg("-prune", 0) != 1); - obj.pushKV("automatic_pruning", automatic_pruning); - if (automatic_pruning) { - obj.pushKV("prune_target_size", nPruneTarget); - } - } - - const Consensus::Params& consensusParams = Params().GetConsensus(0); + obj.push_back(Pair("chain", Params().NetworkIDString())); + obj.push_back(Pair("blocks", (int)chainActive.Height())); + obj.push_back(Pair("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1)); + obj.push_back(Pair("bestblockhash", chainActive.Tip()->GetBlockHash().GetHex())); + obj.push_back(Pair("difficulty", (double)GetDifficulty())); + obj.push_back(Pair("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast())); + obj.push_back(Pair("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip()))); + obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); + obj.push_back(Pair("pruned", fPruneMode)); + + const Consensus::Params& consensusParams = Params().GetConsensus(); CBlockIndex* tip = chainActive.Tip(); UniValue softforks(UniValue::VARR); UniValue bip9_softforks(UniValue::VOBJ); @@ -1266,9 +1181,17 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) softforks.push_back(SoftForkDesc("bip65", 4, tip, consensusParams)); BIP9SoftForkDescPushBack(bip9_softforks, "csv", consensusParams, Consensus::DEPLOYMENT_CSV); BIP9SoftForkDescPushBack(bip9_softforks, "segwit", consensusParams, Consensus::DEPLOYMENT_SEGWIT); - obj.pushKV("softforks", softforks); - obj.pushKV("bip9_softforks", bip9_softforks); - obj.pushKV("warnings", GetWarnings("statusbar")); + obj.push_back(Pair("softforks", softforks)); + obj.push_back(Pair("bip9_softforks", bip9_softforks)); + + if (fPruneMode) + { + CBlockIndex *block = chainActive.Tip(); + while (block && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) + block = block->pprev; + + obj.push_back(Pair("pruneheight", block->nHeight)); + } return obj; } @@ -1290,7 +1213,7 @@ struct CompareBlocksByHeight UniValue getchaintips(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getchaintips\n" "Return information about all known tips in the block tree," " including the main chain as well as orphaned branches.\n" @@ -1356,13 +1279,13 @@ UniValue getchaintips(const JSONRPCRequest& request) BOOST_FOREACH(const CBlockIndex* block, setTips) { UniValue obj(UniValue::VOBJ); - obj.pushKV("height", block->nHeight); - obj.pushKV("hash", block->phashBlock->GetHex()); + obj.push_back(Pair("height", block->nHeight)); + obj.push_back(Pair("hash", block->phashBlock->GetHex())); const int branchLen = block->nHeight - chainActive.FindFork(block)->nHeight; - obj.pushKV("branchlen", branchLen); + obj.push_back(Pair("branchlen", branchLen)); - string status; + std::string status; if (chainActive.Contains(block)) { // This block is part of the currently active chain. status = "active"; @@ -1382,7 +1305,7 @@ UniValue getchaintips(const JSONRPCRequest& request) // No clue. status = "unknown"; } - obj.pushKV("status", status); + obj.push_back(Pair("status", status)); res.push_back(obj); } @@ -1393,12 +1316,12 @@ UniValue getchaintips(const JSONRPCRequest& request) UniValue mempoolInfoToJSON() { UniValue ret(UniValue::VOBJ); - ret.pushKV("size", (int64_t) mempool.size()); - ret.pushKV("bytes", (int64_t) mempool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t) mempool.DynamicMemoryUsage()); + ret.push_back(Pair("size", (int64_t) mempool.size())); + ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); + ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); size_t maxmempool = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - ret.pushKV("maxmempool", (int64_t) maxmempool); - ret.pushKV("mempoolminfee", ValueFromAmount(mempool.GetMinFee(maxmempool).GetFeePerK())); + ret.push_back(Pair("maxmempool", (int64_t) maxmempool)); + ret.push_back(Pair("mempoolminfee", ValueFromAmount(mempool.GetMinFee(maxmempool).GetFeePerK()))); return ret; } @@ -1406,7 +1329,7 @@ UniValue mempoolInfoToJSON() UniValue getmempoolinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw runtime_error( + throw std::runtime_error( "getmempoolinfo\n" "\nReturns details on the active state of the TX memory pool.\n" "\nResult:\n" @@ -1428,7 +1351,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) UniValue preciousblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "preciousblock \"blockhash\"\n" "\nTreats a block as if it were received before others with the same work.\n" "\nA later preciousblock call can override the effect of an earlier one.\n" @@ -1466,7 +1389,7 @@ UniValue preciousblock(const JSONRPCRequest& request) UniValue invalidateblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "invalidateblock \"blockhash\"\n" "\nPermanently marks a block as invalid, as if it violated a consensus rule.\n" "\nArguments:\n" @@ -1504,7 +1427,7 @@ UniValue invalidateblock(const JSONRPCRequest& request) UniValue reconsiderblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw runtime_error( + throw std::runtime_error( "reconsiderblock \"blockhash\"\n" "\nRemoves invalidity status of a block and its descendants, reconsider them for activation.\n" "This can be used to undo the effects of invalidateblock.\n" @@ -1538,326 +1461,68 @@ UniValue reconsiderblock(const JSONRPCRequest& request) return NullUniValue; } -template -static T CalculateTruncatedMedian(std::vector& scores) -{ - size_t size = scores.size(); - if (size == 0) { - return 0; - } - - std::sort(scores.begin(), scores.end()); - if (size % 2 == 0) { - return (scores[size / 2 - 1] + scores[size / 2]) / 2; - } else { - return scores[size / 2]; - } -} - -void CalculatePercentilesBySize(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector>& scores, int64_t total_size) -{ - if (scores.empty()) { - return; - } - - std::sort(scores.begin(), scores.end()); - - // 10th, 25th, 50th, 75th, and 90th percentile weight units. - const double weights[NUM_GETBLOCKSTATS_PERCENTILES] = { - total_size / 10.0, total_size / 4.0, total_size / 2.0, (total_size * 3.0) / 4.0, (total_size * 9.0) / 10.0 - }; - - int64_t next_percentile_index = 0; - int64_t cumulative_weight = 0; - for (const auto& element : scores) { - cumulative_weight += element.second; - while (next_percentile_index < NUM_GETBLOCKSTATS_PERCENTILES && cumulative_weight >= weights[next_percentile_index]) { - result[next_percentile_index] = element.first; - ++next_percentile_index; - } - } - - // Fill any remaining percentiles with the last value. - for (int64_t i = next_percentile_index; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) { - result[i] = scores.back().first; - } -} - -template -static inline bool SetHasKeys(const std::set& set) {return false;} -template -static inline bool SetHasKeys(const std::set& set, const Tk& key, const Args&... args) +UniValue getchaintxstats(const JSONRPCRequest& request) { - return (set.count(key) != 0) || SetHasKeys(set, args...); -} - -// outpoint (needed for the utxo index) + nHeight + fCoinBase -static constexpr size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) + sizeof(bool); - -static UniValue getblockstats(const JSONRPCRequest& request) -{ - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { + if (request.fHelp || request.params.size() > 2) throw std::runtime_error( - "getblockstats hash ( stats )\n" - "\nCompute per block statistics for a given window. All amounts are in koinus.\n" - "It won't work for some heights with pruning.\n" + "getchaintxstats ( nblocks blockhash )\n" + "\nCompute statistics about the total number and rate of transactions in the chain.\n" "\nArguments:\n" - "1. \"hash\" (string, required) The block hash of the target block\n" - "2. \"stats\" (array, optional) Values to plot, by default all values (see result below)\n" - " [\n" - " \"height\", (string, optional) Selected statistic\n" - " \"time\", (string, optional) Selected statistic\n" - " ,...\n" - " ]\n" + "1. nblocks (numeric, optional) Size of the window in number of blocks (default: one month).\n" + "2. \"blockhash\" (string, optional) The hash of the block that ends the window.\n" "\nResult:\n" - "{ (json object)\n" - " \"avgfee\": xxxxx, (numeric) Average fee in the block\n" - " \"avgfeerate\": xxxxx, (numeric) Average feerate (in koinu per byte)\n" - " \"avgtxsize\": xxxxx, (numeric) Average transaction size\n" - " \"blockhash\": xxxxx, (string) The block hash (to check for potential reorgs)\n" - " \"dustouts\": xxxxx, (numeric) Number of outputs under the dust limit (excluding coinbase and OP_RETURN)\n" - " \"feerate_percentiles\": [ (array of numeric) Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in koinu per byte)\n" - " \"10th_percentile_feerate\", (numeric) The 10th percentile feerate\n" - " \"25th_percentile_feerate\", (numeric) The 25th percentile feerate\n" - " \"50th_percentile_feerate\", (numeric) The 50th percentile feerate\n" - " \"75th_percentile_feerate\", (numeric) The 75th percentile feerate\n" - " \"90th_percentile_feerate\", (numeric) The 90th percentile feerate\n" - " ],\n" - " \"height\": xxxxx, (numeric) The height of the block\n" - " \"ins\": xxxxx, (numeric) The number of inputs (excluding coinbase)\n" - " \"maxfee\": xxxxx, (numeric) Maximum fee in the block\n" - " \"maxfeerate\": xxxxx, (numeric) Maximum feerate (in koinu per byte)\n" - " \"maxoutamount\": xxxxx, (numeric) Maximum output value (excluding coinbase and OP_RETURN)\n" - " \"maxtxsize\": xxxxx, (numeric) Maximum transaction size\n" - " \"medianfee\": xxxxx, (numeric) Truncated median fee in the block\n" - " \"mediantime\": xxxxx, (numeric) The block median time past\n" - " \"mediantxsize\": xxxxx, (numeric) Truncated median transaction size\n" - " \"minfee\": xxxxx, (numeric) Minimum fee in the block\n" - " \"minfeerate\": xxxxx, (numeric) Minimum feerate (in koinu per byte)\n" - " \"minoutamount\": xxxxx, (numeric) Minimum output value (excluding coinbase and OP_RETURN)\n" - " \"mintxsize\": xxxxx, (numeric) Minimum transaction size\n" - " \"outs\": xxxxx, (numeric) The number of outputs\n" - " \"subsidy\": xxxxx, (numeric) The block subsidy\n" - " \"time\": xxxxx, (numeric) The block time\n" - " \"total_out\": xxxxx, (numeric) Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])\n" - " \"total_size\": xxxxx, (numeric) Total size of all non-coinbase transactions\n" - " \"total_weight\": xxxxx, (numeric) Total weight of all non-coinbase transactions\n" - " \"totalfee\": xxxxx, (numeric) The fee total\n" - " \"txs\": xxxxx, (numeric) The number of transactions (including coinbase)\n" - " \"utxo_increase\": xxxxx, (numeric) The increase/decrease in the number of unspent outputs\n" - " \"utxo_size_inc\": xxxxx, (numeric) The increase/decrease in size for the utxo index (not discounting op_return and similar)\n" + "{\n" + " \"time\": xxxxx, (numeric) The timestamp for the statistics in UNIX format.\n" + " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" + " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window.\n" "}\n" "\nExamples:\n" - + HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") - + HelpExampleRpc("getblockstats", R"("00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09", ["minfeerate","avgfeerate"])") + + HelpExampleCli("getchaintxstats", "") + + HelpExampleRpc("getchaintxstats", "2016") ); - } - - LOCK(cs_main); - - CBlockIndex* pindex; - const uint256 hash = ParseHashV(request.params[0], "hash"); - if (mapBlockIndex.count(hash) == 0) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - pindex = mapBlockIndex[hash]; + const CBlockIndex* pindex; + int blockcount = 30 * 24 * 60 * 60 / Params().GetConsensus().nPowTargetSpacing; // By default: 1 month - if (!chainActive.Contains(pindex)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); + if (request.params.size() > 0 && !request.params[0].isNull()) { + blockcount = request.params[0].get_int(); } - assert(pindex != nullptr); - - std::set stats; - if (!request.params[1].isNull()) { - const UniValue stats_univalue = request.params[1].get_array(); - for (unsigned int i = 0; i < stats_univalue.size(); i++) { - const std::string stat = stats_univalue[i].get_str(); - stats.insert(stat); - } + bool havehash = request.params.size() > 1 && !request.params[1].isNull(); + uint256 hash; + if (havehash) { + hash = uint256S(request.params[1].get_str()); } - const CBlock block = GetBlockChecked(pindex); - const CBlockUndo blockUndo = GetUndoChecked(pindex); - - const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default) - const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0; - const bool do_medianfee = do_all || stats.count("medianfee") != 0; - const bool do_feerate_percentiles = do_all || stats.count("feerate_percentiles") != 0; - const bool do_duststats = do_all || SetHasKeys(stats, "minoutamount", "maxoutamount", "dustouts"); - const bool loop_inputs = do_all || do_medianfee || do_feerate_percentiles || - SetHasKeys(stats, "utxo_size_inc", "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate", "subsidy"); - const bool loop_outputs = do_all || loop_inputs || do_duststats || stats.count("total_out"); - const bool do_calculate_size = do_all || do_mediantxsize || - SetHasKeys(stats, "total_size", "avgtxsize", "mintxsize", "maxtxsize", "avgfeerate", "avgfeerate", "feerate_percentiles", "minfeerate", "maxfeerate"); - const bool do_calculate_weight = do_all || SetHasKeys(stats, "total_weight"); - - CAmount maxfee = 0; - CAmount maxfeerate = 0; - CAmount maxoutamount = 0; - CAmount minfee = MAX_MONEY; - CAmount minfeerate = MAX_MONEY; - CAmount minoutamount = MAX_MONEY; - CAmount total_out = 0; - CAmount totalfee = 0; - int64_t dustouts = 0; - int64_t inputs = 0; - int64_t maxtxsize = 0; - int64_t mintxsize = MAX_BLOCK_SERIALIZED_SIZE; - int64_t outputs = 0; - int64_t total_size = 0; - int64_t total_weight = 0; - int64_t utxo_size_inc = 0; - std::vector fee_array; - std::vector> feerate_array; - std::vector txsize_array; - - bool witnessEnabled = IsWitnessEnabled(pindex, Params().GetConsensus(pindex->nHeight)); - txnouttype whichType; - - for (size_t i = 0; i < block.vtx.size(); ++i) { - const auto& tx = block.vtx.at(i); - outputs += tx->vout.size(); - - CAmount tx_total_out = 0; - if (loop_outputs) { - for (const CTxOut& out : tx->vout) { - tx_total_out += out.nValue; - utxo_size_inc += GetSerializeSize(out, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; - if (do_duststats) { - if (tx->IsCoinBase()) { - continue; - } - - if (::IsStandard(out.scriptPubKey, whichType, witnessEnabled)) { - if (whichType == TX_NULL_DATA) { - continue; - } - } - - if (out.nValue > maxoutamount) { - maxoutamount = out.nValue; - } - if (out.nValue < minoutamount) { - minoutamount = out.nValue; - } - if (out.nValue < nDustLimit) { - dustouts += 1; - } - } - } - } - - if (tx->IsCoinBase()) { - continue; - } - - inputs += tx->vin.size(); // Don't count coinbase's fake input - total_out += tx_total_out; // Don't count coinbase reward - - int64_t tx_size = 0; - if (do_calculate_size) { - - tx_size = tx->GetTotalSize(); - if (do_mediantxsize) { - txsize_array.push_back(tx_size); - } - maxtxsize = std::max(maxtxsize, tx_size); - mintxsize = std::min(mintxsize, tx_size); - total_size += tx_size; - } - - int64_t weight = 0; - if (do_calculate_weight) { - weight = GetTransactionWeight(*tx); - total_weight += weight; - } - - if (loop_inputs) { - CAmount tx_total_in = 0; - const auto& txundo = blockUndo.vtxundo.at(i - 1); - for (const CTxInUndo& coin: txundo.vprevout) { - const CTxOut& prevoutput = coin.txout; - - tx_total_in += prevoutput.nValue; - utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; - } - - CAmount txfee = tx_total_in - tx_total_out; - assert(MoneyRange(txfee)); - if (do_medianfee) { - fee_array.push_back(txfee); + { + LOCK(cs_main); + if (havehash) { + auto it = mapBlockIndex.find(hash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - maxfee = std::max(maxfee, txfee); - minfee = std::min(minfee, txfee); - totalfee += txfee; - - CAmount feerate = tx_size ? txfee / tx_size : 0; // Unit: koinu/byte - if (do_feerate_percentiles) { - feerate_array.emplace_back(std::make_pair(feerate, tx_size)); + pindex = it->second; + if (!chainActive.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); } - maxfeerate = std::max(maxfeerate, feerate); - minfeerate = std::min(minfeerate, feerate); + } else { + pindex = chainActive.Tip(); } } - CAmount feerate_percentiles[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 }; - CalculatePercentilesBySize(feerate_percentiles, feerate_array, total_size); - - UniValue feerates_res(UniValue::VARR); - for (int64_t i = 0; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) { - feerates_res.push_back(feerate_percentiles[i]); + if (blockcount < 1 || blockcount >= pindex->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 1 and the block's height"); } - CAmount subsidy = 0; - for (const auto& tx: block.vtx[0]->vout) { - subsidy += tx.nValue; - } - subsidy = std::max(subsidy - totalfee, (CAmount)0); - - UniValue ret_all(UniValue::VOBJ); - ret_all.pushKV("avgfee", (block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0); - ret_all.pushKV("avgfeerate", total_size ? totalfee / total_size : 0); // Unit: koinu/byte - ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0); - ret_all.pushKV("blockhash", pindex->GetBlockHash().GetHex()); - ret_all.pushKV("dustouts", dustouts); - ret_all.pushKV("feerate_percentiles", feerates_res); - ret_all.pushKV("height", (int64_t)pindex->nHeight); - ret_all.pushKV("ins", inputs); - ret_all.pushKV("maxfee", maxfee); - ret_all.pushKV("maxfeerate", maxfeerate); - ret_all.pushKV("maxoutamount", maxoutamount); - ret_all.pushKV("maxtxsize", maxtxsize); - ret_all.pushKV("medianfee", CalculateTruncatedMedian(fee_array)); - ret_all.pushKV("mediantime", pindex->GetMedianTimePast()); - ret_all.pushKV("mediantxsize", CalculateTruncatedMedian(txsize_array)); - ret_all.pushKV("minfee", (minfee == MAX_MONEY) ? 0 : minfee); - ret_all.pushKV("minfeerate", (minfeerate == MAX_MONEY) ? 0 : minfeerate); - ret_all.pushKV("minoutamount", (minoutamount == MAX_MONEY) ? 0 : minoutamount); - ret_all.pushKV("mintxsize", mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize); - ret_all.pushKV("outs", outputs); - ret_all.pushKV("subsidy", subsidy); - ret_all.pushKV("time", pindex->GetBlockTime()); - ret_all.pushKV("total_out", total_out); - ret_all.pushKV("total_size", total_size); - ret_all.pushKV("total_weight", total_weight); - ret_all.pushKV("totalfee", totalfee); - ret_all.pushKV("txs", (int64_t)block.vtx.size()); - ret_all.pushKV("utxo_increase", outputs - inputs); - ret_all.pushKV("utxo_size_inc", utxo_size_inc); - - if (do_all) { - return ret_all; - } + const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; UniValue ret(UniValue::VOBJ); - for (const std::string& stat : stats) { - const UniValue& value = ret_all[stat]; - if (value.isNull()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid selected statistic %s", stat)); - } - ret.pushKV(stat, value); - } + ret.push_back(Pair("time", (int64_t)pindex->nTime)); + ret.push_back(Pair("txcount", (int64_t)pindex->nChainTx)); + ret.push_back(Pair("txrate", ((double)nTxDiff) / nTimeDiff)); + return ret; } @@ -1865,10 +1530,10 @@ static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} }, - { "blockchain", "getblockstats", &getblockstats, true, {"hash", "stats"} }, + { "blockchain", "getchaintxstats", &getchaintxstats, true, {"nblocks", "blockhash"} }, { "blockchain", "getbestblockhash", &getbestblockhash, true, {} }, { "blockchain", "getblockcount", &getblockcount, true, {} }, - { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity"} }, + { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity|verbose"} }, { "blockchain", "getblockhash", &getblockhash, true, {"height"} }, { "blockchain", "getblockheader", &getblockheader, true, {"blockhash","verbose"} }, { "blockchain", "getchaintips", &getchaintips, true, {} }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 0459b3902bf..8ac5260160c 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -327,9 +327,10 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins(oneTxid, coins) && coins.nHeight > 0 && coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsPruned() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + pblockindex = chainActive[coin.nHeight]; + } } if (pblockindex == NULL) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index dcde4f1ded3..8d519c644b9 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,6 +8,7 @@ #include "undo.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" +#include "test/test_random.h" #include "validation.h" #include "consensus/validation.h" @@ -33,27 +34,27 @@ bool operator==(const Coin &a, const Coin &b) { class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map map_; + std::map map_; public: - bool GetCoins(const uint256& txid, CCoins& coins) const + bool GetCoins(const COutPoint& outpoint, Coin& coin) const { - std::map::const_iterator it = map_.find(txid); + std::map::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && InsecureRandBool() == 0) { + coin = it->second; + if (coin.IsPruned() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256& txid) const + bool HaveCoins(const COutPoint& outpoint) const { - CCoins coins; - return GetCoins(txid, coins); + Coin coin; + return GetCoins(outpoint, coin); } uint256 GetBestBlock() const { return hashBestBlock_; } @@ -64,7 +65,7 @@ class CCoinsViewTest : public CCoinsView if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && InsecureRandRange(3) == 0) { + if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -80,7 +81,7 @@ class CCoinsViewTest : public CCoinsView class CCoinsViewCacheTest : public CCoinsViewCache { public: - CCoinsViewCacheTest(CCoinsView* base) : CCoinsViewCache(base) {} + CCoinsViewCacheTest(CCoinsView* _base) : CCoinsViewCache(_base) {} void SelfTest() const { @@ -105,7 +106,7 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // This is a large randomized insert/remove simulation test on a variable-size // stack of caches on top of CCoinsViewTest. // -// It will randomly create/update/delete CCoins entries to a tip of caches, with +// It will randomly create/update/delete Coin entries to a tip of caches, with // txids picked from a limited list of random 256-bit hashes. Occasionally, a // new tip is added to the stack of caches, or the tip is flushed and removed. // @@ -123,7 +124,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) bool missed_an_entry = false; // A simple map to track what we expect the cache stack to represent. - std::map result; + std::map result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -134,46 +135,45 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) std::vector txids; txids.resize(NUM_SIMULATION_ITERATIONS / 8); for (unsigned int i = 0; i < txids.size(); i++) { - txids[i] = InsecureRand256(); + txids[i] = GetRandHash(); } for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { // Do a random modification. { - uint256 txid = txids[InsecureRandRange(500) % txids.size()]; // txid we're going to modify in this iteration. - CCoins& coins = result[txid]; + uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. + Coin& coin = result[COutPoint(txid, 0)]; const Coin& entry = stack.back()->AccessCoin(COutPoint(txid, 0)); - BOOST_CHECK((entry.IsPruned() && coins.IsPruned()) || entry == Coin(coins.vout[0], coins.nHeight, coins.fCoinBase)); + BOOST_CHECK(coin == entry); - if (InsecureRandRange(5) == 0 || coins.IsPruned()) { - if (coins.IsPruned()) { + if (insecure_rand() % 5 == 0 || coin.IsPruned()) { + if (coin.IsPruned()) { added_an_entry = true; } else { updated_an_entry = true; } - coins.vout.resize(1); - coins.vout[0].nValue = InsecureRand32(); + coin.out.nValue = insecure_rand(); + coin.nHeight = 1; } else { - coins.Clear(); + coin.Clear(); removed_an_entry = true; } - if (coins.IsPruned()) { + if (coin.IsPruned()) { stack.back()->SpendCoin(COutPoint(txid, 0)); } else { - stack.back()->AddCoin(COutPoint(txid, 0), Coin(coins.vout[0], coins.nHeight, coins.fCoinBase), true); + stack.back()->AddCoin(COutPoint(txid, 0), Coin(coin), true); } } // Once every 1000 iterations and at the end, verify the full cache. - if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - found_an_entry = true; - } else { - BOOST_CHECK(it->second.IsPruned()); + if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { + for (auto it = result.begin(); it != result.end(); it++) { + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(coin == it->second); + if (coin.IsPruned()) { missed_an_entry = true; + } else { + found_an_entry = true; } } BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { @@ -181,22 +181,22 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache - if (stack.size() > 1 && InsecureRandBool() == 0) { - unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (stack.size() > 1 && insecure_rand() % 2 == 0) { + unsigned int flushIndex = insecure_rand() % (stack.size() - 1); stack[flushIndex]->Flush(); } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, change the cache stack. - if (stack.size() > 0 && InsecureRandBool() == 0) { + if (stack.size() > 0 && insecure_rand() % 2 == 0) { //Remove the top cache stack.back()->Flush(); delete stack.back(); stack.pop_back(); } - if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { + if (stack.size() == 0 || (stack.size() < 4 && insecure_rand() % 2)) { //Add a new cache CCoinsView* tip = &base; if (stack.size() > 0) { @@ -228,19 +228,19 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(missed_an_entry); } -typedef std::tuple TxData; // Store of all necessary tx and undo data for next test -std::map alltxs; - -TxData &FindRandomFrom(const std::set &txidset) { - assert(txidset.size()); - std::set::iterator txIt = txidset.lower_bound(InsecureRand256()); - if (txIt == txidset.end()) { - txIt = txidset.begin(); +typedef std::map> UtxoData; +UtxoData utxoData; + +UtxoData::iterator FindRandomFrom(const std::set &utxoSet) { + assert(utxoSet.size()); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(GetRandHash(), 0)); + if (utxoSetIt == utxoSet.end()) { + utxoSetIt = utxoSet.begin(); } - std::map::iterator txdit = alltxs.find(*txIt); - assert(txdit != alltxs.end()); - return txdit->second; + auto utxoDataIt = utxoData.find(*utxoSetIt); + assert(utxoDataIt != utxoData.end()); + return utxoDataIt; } @@ -248,12 +248,12 @@ TxData &FindRandomFrom(const std::set &txidset) { // except the emphasis is on testing the functionality of UpdateCoins // random txs are created and UpdateCoins is used to update the cache stack // In particular it is tested that spending a duplicate coinbase tx -// has the expected effect (the other duplicate is overwritten at all cache levels) +// has the expected effect (the other duplicate is overwitten at all cache levels) BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { bool spent_a_duplicate_coinbase = false; // A simple map to track what we expect the cache stack to represent. - std::map result; + std::map result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -261,13 +261,13 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Track the txids we've used in various sets - std::set coinbaseids; - std::set disconnectedids; - std::set duplicateids; - std::set utxoset; + std::set coinbaseids; + std::set disconnectedids; + std::set duplicateids; + std::set utxoset; for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { - uint32_t randiter = InsecureRand32(); + uint32_t randiter = insecure_rand(); // 19/20 txs add a new transaction if (randiter % 20 < 19) { @@ -275,23 +275,23 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vin.resize(1); tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate - unsigned int height = InsecureRand32(); - CCoins oldcoins; + unsigned int height = insecure_rand(); + Coin oldcoins; // 2/20 times create a new coinbase if (randiter % 20 < 2 || coinbaseids.size() < 10) { // 1/10 of those times create a duplicate coinbase - if (InsecureRandRange(10) == 0 && coinbaseids.size()) { - TxData &txd = FindRandomFrom(coinbaseids); + if (insecure_rand() % 10 == 0 && coinbaseids.size()) { + auto utxod = FindRandomFrom(coinbaseids); // Reuse the exact same coinbase - tx = std::get<0>(txd); + tx = std::get<0>(utxod->second); // shouldn't be available for reconnection if its been duplicated - disconnectedids.erase(tx.GetHash()); + disconnectedids.erase(utxod->first); - duplicateids.insert(tx.GetHash()); + duplicateids.insert(utxod->first); } else { - coinbaseids.insert(tx.GetHash()); + coinbaseids.insert(COutPoint(tx.GetHash(), 0)); } assert(CTransaction(tx).IsCoinBase()); } @@ -299,85 +299,82 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // 17/20 times reconnect previous or add a regular tx else { - uint256 prevouthash; + COutPoint prevout; // 1/20 times reconnect a previously disconnected tx if (randiter % 20 == 2 && disconnectedids.size()) { - TxData &txd = FindRandomFrom(disconnectedids); - tx = std::get<0>(txd); - prevouthash = tx.vin[0].prevout.hash; - if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevouthash)) { - disconnectedids.erase(tx.GetHash()); + auto utxod = FindRandomFrom(disconnectedids); + tx = std::get<0>(utxod->second); + prevout = tx.vin[0].prevout; + if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevout)) { + disconnectedids.erase(utxod->first); continue; } // If this tx is already IN the UTXO, then it must be a coinbase, and it must be a duplicate - if (utxoset.count(tx.GetHash())) { + if (utxoset.count(utxod->first)) { assert(CTransaction(tx).IsCoinBase()); - assert(duplicateids.count(tx.GetHash())); + assert(duplicateids.count(utxod->first)); } - disconnectedids.erase(tx.GetHash()); + disconnectedids.erase(utxod->first); } // 16/20 times create a regular tx else { - TxData &txd = FindRandomFrom(utxoset); - prevouthash = std::get<0>(txd).GetHash(); + auto utxod = FindRandomFrom(utxoset); + prevout = utxod->first; // Construct the tx to spend the coins of prevouthash - tx.vin[0].prevout.hash = prevouthash; - tx.vin[0].prevout.n = 0; + tx.vin[0].prevout = prevout; assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevouthash]; + oldcoins = result[prevout]; // Update the expected result of prevouthash to know these coins are spent - result[prevouthash].Clear(); + result[prevout].Clear(); - utxoset.erase(prevouthash); + utxoset.erase(prevout); // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) { + if (duplicateids.count(prevout)) { spent_a_duplicate_coinbase = true; } } // Update the expected result to know about the new output coins - result[tx.GetHash()].FromTx(tx, height); + assert(tx.vout.size() == 1); + const COutPoint outpoint(tx.GetHash(), 0); + result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); // Call UpdateCoins on the top cache CTxUndo undo; UpdateCoins(tx, *(stack.back()), undo, height); // Update the utxo set for future spends - utxoset.insert(tx.GetHash()); + utxoset.insert(outpoint); // Track this tx and undo info to use later - alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,oldcoins)); } else if (utxoset.size()) { //1/20 times undo a previous transaction - TxData &txd = FindRandomFrom(utxoset); - - CTransaction &tx = std::get<0>(txd); - CTxUndo &undo = std::get<1>(txd); - CCoins &origcoins = std::get<2>(txd); + auto utxod = FindRandomFrom(utxoset); - uint256 undohash = tx.GetHash(); + CTransaction &tx = std::get<0>(utxod->second); + CTxUndo &undo = std::get<1>(utxod->second); + Coin &origcoins = std::get<2>(utxod->second); // Update the expected result // Remove new outputs - result[undohash].Clear(); + result[utxod->first].Clear(); // If not coinbase restore prevout if (!tx.IsCoinBase()) { - result[tx.vin[0].prevout.hash] = origcoins; + result[tx.vin[0].prevout] = origcoins; } // Disconnect the tx from the current UTXO // See code in DisconnectBlock // remove outputs - { - stack.back()->SpendCoin(COutPoint(undohash, 0)); - } + stack.back()->SpendCoin(utxod->first); // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; @@ -385,41 +382,37 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection - disconnectedids.insert(undohash); + disconnectedids.insert(utxod->first); // Update the utxoset - utxoset.erase(undohash); + utxoset.erase(utxod->first); if (!tx.IsCoinBase()) - utxoset.insert(tx.vin[0].prevout.hash); + utxoset.insert(tx.vin[0].prevout); } // Once every 1000 iterations and at the end, verify the full cache. - if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - } else { - BOOST_CHECK(it->second.IsPruned()); - } + if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { + for (auto it = result.begin(); it != result.end(); it++) { + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(coin == it->second); } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache - if (stack.size() > 1 && InsecureRandBool() == 0) { - unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (stack.size() > 1 && insecure_rand() % 2 == 0) { + unsigned int flushIndex = insecure_rand() % (stack.size() - 1); stack[flushIndex]->Flush(); } } - if (InsecureRandRange(100) == 0) { + if (insecure_rand() % 100 == 0) { // Every 100 iterations, change the cache stack. - if (stack.size() > 0 && InsecureRandBool() == 0) { + if (stack.size() > 0 && insecure_rand() % 2 == 0) { stack.back()->Flush(); delete stack.back(); stack.pop_back(); } - if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { + if (stack.size() == 0 || (stack.size() < 4 && insecure_rand() % 2)) { CCoinsView* tip = &base; if (stack.size() > 0) { tip = stack.back(); @@ -442,50 +435,36 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); - CCoins cc1; + CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + Coin cc1; ss1 >> cc1; BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); - BOOST_CHECK_EQUAL(cc1.vout.size(), 2); - BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); - BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); - BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); - BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + BOOST_CHECK_EQUAL(cc1.out.nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); - CCoins cc2; + CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + Coin cc2; ss2 >> cc2; BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); - BOOST_CHECK_EQUAL(cc2.vout.size(), 17); - for (int i = 0; i < 17; i++) { - BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); - } - BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); - BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + BOOST_CHECK_EQUAL(cc2.out.nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ssx(SER_DISK, CLIENT_VERSION); - BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); - - CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); - CCoins cc3; + CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + Coin cc3; ss3 >> cc3; BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); - BOOST_CHECK_EQUAL(cc3.vout.size(), 1); - BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); - BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); - BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + BOOST_CHECK_EQUAL(cc3.out.nValue, 0); + BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); try { - CCoins cc4; + Coin cc4; ss4 >> cc4; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { @@ -496,17 +475,16 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); - CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { - CCoins cc5; + Coin cc5; ss5 >> cc5; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { } } -const static uint256 TXID; -const static COutPoint OUTPOINT = {uint256(), 0}; +const static COutPoint OUTPOINT; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -521,15 +499,15 @@ const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; const static auto CLEAN_FLAGS = {char(0), FRESH}; const static auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins& coins) +void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsPruned()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + coin.out.nValue = value; + coin.nHeight = 1; + assert(!coin.IsPruned()); } } @@ -543,24 +521,22 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) CCoinsCacheEntry entry; entry.flags = flags; SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); return inserted.first->second.coins.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coins.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index c4983f6f5cc..6dc850b6ab9 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -169,8 +169,8 @@ int main(int argc, char **argv) { try { - CCoins block; - ds >> block; + Coin coin; + ds >> coin; } catch (const std::ios_base::failure& e) {return 0;} break; } diff --git a/src/txdb.cpp b/src/txdb.cpp index c56935ee936..505ceb6089d 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,17 +25,40 @@ static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinsEntry { + COutPoint* outpoint; + char key; + CoinsEntry(const COutPoint* ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} + + template + void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template + void Unserialize(Stream& s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; + +} CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoins(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinsEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoins(const COutPoint &outpoint) const { + return db.Exists(CoinsEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -50,10 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { + CoinsEntry entry(&it->first); if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); + batch.Erase(entry); else - batch.Write(std::make_pair(DB_COINS, it->first), it->second.coins); + batch.Write(entry, it->second.coins); changed++; } count++; @@ -63,13 +88,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); - return db.WriteBatch(batch); + bool ret = db.WriteBatch(batch); + LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); + return ret; } size_t CCoinsViewDB::EstimateSize() const { - return db.EstimateSize(DB_COINS, (char)(DB_COINS+1)); + return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -101,29 +127,31 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record if (i->pcursor->Valid()) { - i->pcursor->GetKey(i->keyTmp); + CoinsEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; } else { i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false } return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { - return pcursor->GetValue(coins); + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const @@ -133,14 +161,18 @@ unsigned int CCoinsViewDBCursor::GetValueSize() const bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinsEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync(const std::vector >& fileInfo, int nLastFile, const std::vector& blockinfo) { @@ -206,10 +238,8 @@ bool CBlockTreeDB::LoadBlockIndexGuts(std::functionnStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; - /* Bitcoin checks the PoW here. We don't do this because - the CDiskBlockIndex does not contain the auxpow. - This check isn't important, since the data on disk should - already be valid and can be trusted. */ + if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, Params().GetConsensus())) + return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString()); pcursor->Next(); } else { diff --git a/src/txdb.h b/src/txdb.h index c523d12c886..12c08664896 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -74,11 +74,11 @@ class CCoinsViewDB : public CCoinsView public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool GetCoins(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoins(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; size_t EstimateSize() const override; }; @@ -89,8 +89,8 @@ class CCoinsViewDBCursor: public CCoinsViewCursor public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -100,7 +100,7 @@ class CCoinsViewDBCursor: public CCoinsViewCursor CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr pcursor; - std::pair keyTmp; + std::pair keyTmp; friend class CCoinsViewDB; }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 3e310058b85..328573da498 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,14 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2018-2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "txmempool.h" -#include "chainparams.h" -#include "clientversion.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "policy/policy.h" @@ -18,25 +16,19 @@ #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, double _entryPriority, unsigned int _entryHeight, - CAmount _inChainInputValue, + int64_t _nTime, unsigned int _entryHeight, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): - tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), - inChainInputValue(_inChainInputValue), + tx(_tx), nFee(_nFee), nTime(_nTime), entryHeight(_entryHeight), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); - nModSize = tx->CalculateModifiedSize(GetTxSize()); nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); nModFeesWithDescendants = nFee; - CAmount nValueIn = tx->GetValueOut()+nFee; - assert(inChainInputValue <= nValueIn); feeDelta = 0; @@ -46,14 +38,9 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe nSigOpCostWithAncestors = sigOpCost; } -double -CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const +CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) { - double deltaPriority = ((double)(currentHeight-entryHeight)*inChainInputValue)/nModSize; - double dResult = entryPriority + deltaPriority; - if (dResult < 0) // This should only happen if it was called with a height below entry height - dResult = 0; - return dResult; + *this = other; } void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) @@ -120,7 +107,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendan // vHashesToUpdate is the set of transaction hashes from a disconnected block // which has been re-added to the mempool. -// for each entry, look for descendants that are outside hashesToUpdate, and +// for each entry, look for descendants that are outside vHashesToUpdate, and // add fee/size information for such descendants to the parent. // for each such descendant, also update the ancestor state to include the parent. void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashesToUpdate) @@ -135,7 +122,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashes // accounted for in the state of their ancestors) std::set setAlreadyIncluded(vHashesToUpdate.begin(), vHashesToUpdate.end()); - // Iterate in reverse, so that whenever we are looking at a transaction + // Iterate in reverse, so that whenever we are looking at at a transaction // we are sure that all in-mempool descendants have already been processed. // This maximizes the benefit of the descendant cache and guarantees that // setMemPoolChildren will be updated, an assumption made in @@ -345,8 +332,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : - nTransactionsUpdated(0) +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -354,13 +341,6 @@ CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(_minReasonableRelayFee); -} - -CTxMemPool::~CTxMemPool() -{ - delete minerPolicyEstimator; } bool CTxMemPool::isSpent(const COutPoint& outpoint) @@ -394,11 +374,11 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, // Update transaction for any feeDelta created by PrioritiseTransaction // TODO: refactor so that the fee delta is calculated before inserting // into mapTx. - std::map >::const_iterator pos = mapDeltas.find(hash); + std::map::const_iterator pos = mapDeltas.find(hash); if (pos != mapDeltas.end()) { - const std::pair &deltas = pos->second; - if (deltas.second) { - mapTx.modify(newit, update_fee_delta(deltas.second)); + const CAmount &delta = pos->second; + if (delta) { + mapTx.modify(newit, update_fee_delta(delta)); } } @@ -432,7 +412,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -462,7 +442,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -544,10 +524,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); - int nCoinbaseMaturity = Params().GetConsensus(coins->nHeight).nCoinbaseMaturity; - if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < nCoinbaseMaturity)) { + const Coin &coin = pcoins->AccessCoin(txin.prevout); + if (nCheckFrequency != 0) assert(!coin.IsPruned()); + if (coin.IsPruned() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; } @@ -597,7 +576,7 @@ void CTxMemPool::removeForBlock(const std::vector& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -640,14 +619,13 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (GetRand(std::numeric_limits::max()) >= nCheckFrequency) return; - LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); + LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast(pcoins)); const int64_t nSpendHeight = GetSpendHeight(mempoolDuplicate); - const CChainParams& params = Params(); LOCK(cs); std::list waitingOnDependants; @@ -676,8 +654,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); - assert(coins && coins->IsAvailable(txin.prevout.n)); + assert(pcoins->HaveCoins(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -729,7 +706,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const else { CValidationState state; bool fCheckResult = tx.IsCoinBase() || - Consensus::CheckTxInputs(params, tx, state, mempoolDuplicate, nSpendHeight); + Consensus::CheckTxInputs(tx, state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, 1000000); } @@ -745,7 +722,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { bool fCheckResult = entry->GetTx().IsCoinBase() || - Consensus::CheckTxInputs(params, entry->GetTx(), state, mempoolDuplicate, nSpendHeight); + Consensus::CheckTxInputs(entry->GetTx(), state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); stepsSinceLastRemove = 0; @@ -857,81 +834,15 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -CFeeRate CTxMemPool::estimateFee(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateFee(nBlocks); -} -CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); -} -double CTxMemPool::estimatePriority(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimatePriority(nBlocks); -} -double CTxMemPool::estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartPriority(nBlocks, answerFoundAtBlocks, *this); -} - -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const -{ - try { - LOCK(cs); - fileout << FEEFILE_MIN_BACKCOMPAT_VERSION; - fileout << CLIENT_VERSION; // version that wrote the file - minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) -{ - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - - /* From 1.14.7, only accept fee estimate files created by a version equal to or higher than the - * minimum writer version. - * - * If in the future any of the MIN_FEERATE, MAX_FEERATE or FEE_SPACING in policy/fees.h change, - * or these values become configurable, this logic will need to be adapted. */ - if (nVersionThatWrote < FEEFILE_MIN_COMPAT_VERSION_WRITER) { - return error("CTxMemPool::ReadFeeEstimates(): cannot re-use fee estimate files from version %d, ignoring file (non-fatal)", nVersionThatWrote); - } - - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -void CTxMemPool::PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta) +void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { LOCK(cs); - std::pair &deltas = mapDeltas[hash]; - deltas.first += dPriorityDelta; - deltas.second += nFeeDelta; + CAmount &delta = mapDeltas[hash]; + delta += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { - mapTx.modify(it, update_fee_delta(deltas.second)); + mapTx.modify(it, update_fee_delta(delta)); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); @@ -950,18 +861,17 @@ void CTxMemPool::PrioritiseTransaction(const uint256 hash, const std::string str ++nTransactionsUpdated; } } - LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, FormatMoney(nFeeDelta)); + LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } -void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const +void CTxMemPool::ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const { LOCK(cs); - std::map >::const_iterator pos = mapDeltas.find(hash); + std::map::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) return; - const std::pair &deltas = pos->second; - dPriorityDelta += deltas.first; - nFeeDelta += deltas.second; + const CAmount &delta = pos->second; + nFeeDelta += delta; } void CTxMemPool::ClearPrioritisation(const uint256 hash) @@ -980,20 +890,24 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoins(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } else { + return false; + } } - return (base->GetCoins(txid, coins) && !coins.IsPruned()); + return (base->GetCoins(outpoint, coin) && !coin.IsPruned()); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins(txid); +bool CCoinsViewMemPool::HaveCoins(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoins(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1104,7 +1018,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { } } -void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining) { +void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; @@ -1135,18 +1049,18 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRe if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (exists(txin.prevout.hash)) - continue; - auto iter = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || iter->first->hash != txin.prevout.hash) - pvNoSpendsRemaining->push_back(txin.prevout.hash); + if (exists(txin.prevout.hash)) continue; + if (!mapNextTx.count(txin.prevout)) { + pvNoSpendsRemaining->push_back(txin.prevout); + } } } } } - if (maxFeeRateRemoved > CFeeRate(0)) - LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + if (maxFeeRateRemoved > CFeeRate(0)) { + LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + } } bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLimit) const { @@ -1155,3 +1069,5 @@ bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLi return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 822e5619f52..c86859118a1 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,6 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers -// Copyright (c) 2018-2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,11 +16,11 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "policy/feerate.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" -#undef foreach #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" #include "boost/multi_index/hashed_index.hpp" @@ -31,33 +30,8 @@ class CAutoFile; class CBlockIndex; -/** Minimum client version required to read fee_estimates.dat - * Clients with a lower version than this should ignore the file - */ -static const int FEEFILE_MIN_BACKCOMPAT_VERSION = 1140700; // 1.14.7.0 - -/** Minimum client version required to read fee_estimates.dat - * Files created with a lower version than this are ignored - * - * Due to bucket spacing and min/max changes in 1.14.7, set this - * to 1.14.7.0 exactly. - */ -static const int FEEFILE_MIN_COMPAT_VERSION_WRITER = 1140700; // 1.14.7.0 - -inline double AllowFreeThreshold() -{ - return COIN * 144 / 250; -} - -inline bool AllowFree(double dPriority) -{ - // Large (in bytes) low-priority (new, small-coin) transactions - // need a fee. - return dPriority > AllowFreeThreshold(); -} - -/** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ -static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; +/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ +static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { @@ -86,11 +60,6 @@ class CTxMemPool; * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * - * If updating the descendant state is skipped, we can mark the entry as - * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal nTxSize/ - * nFee+feeDelta. (This can potentially happen during a reorg, where we limit the - * amount of work we're willing to do to avoid consuming too much CPU.) - * */ class CTxMemPoolEntry @@ -99,12 +68,9 @@ class CTxMemPoolEntry CTransactionRef tx; CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) - size_t nModSize; //!< ... and modified size for priority size_t nUsageSize; //!< ... and total memory usage int64_t nTime; //!< Local time when entering the mempool - double entryPriority; //!< Priority when entering the mempool unsigned int entryHeight; //!< Chain height when entering the mempool - CAmount inChainInputValue; //!< Sum of all txin values that are already in blockchain bool spendsCoinbase; //!< keep track of transactions that spend a coinbase int64_t sigOpCost; //!< Total sigop cost int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block @@ -112,9 +78,7 @@ class CTxMemPoolEntry // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these - // descendants as well. if nCountWithDescendants is 0, treat this entry as - // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be - // correct. + // descendants as well. uint64_t nCountWithDescendants; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) @@ -127,17 +91,14 @@ class CTxMemPoolEntry public: CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, double _entryPriority, unsigned int _entryHeight, - CAmount _inChainInputValue, bool spendsCoinbase, + int64_t _nTime, unsigned int _entryHeight, + bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); + CTxMemPoolEntry(const CTxMemPoolEntry& other); + const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } - /** - * Fast calculation of lower bound of current priority as update - * from entry priority. Only inputs that were originally in-chain will age. - */ - double GetPriority(unsigned int currentHeight) const; const CAmount& GetFee() const { return nFee; } size_t GetTxSize() const; size_t GetTxWeight() const { return nTxWeight; } @@ -148,7 +109,7 @@ class CTxMemPoolEntry size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } - // Adjusts the descendant state, if this entry is not dirty. + // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); @@ -241,7 +202,7 @@ struct mempoolentry_txid class CompareTxMemPoolEntryByDescendantScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); @@ -263,7 +224,7 @@ class CompareTxMemPoolEntryByDescendantScore } // Calculate which score to use for an entry (avoiding division). - bool UseDescendantScore(const CTxMemPoolEntry &a) const + bool UseDescendantScore(const CTxMemPoolEntry &a) { double f1 = (double)a.GetModifiedFee() * a.GetSizeWithDescendants(); double f2 = (double)a.GetModFeesWithDescendants() * a.GetTxSize(); @@ -278,7 +239,7 @@ class CompareTxMemPoolEntryByDescendantScore class CompareTxMemPoolEntryByScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) { double f1 = (double)a.GetModifiedFee() * b.GetTxSize(); double f2 = (double)b.GetModifiedFee() * a.GetTxSize(); @@ -292,7 +253,7 @@ class CompareTxMemPoolEntryByScore class CompareTxMemPoolEntryByEntryTime { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) { return a.GetTime() < b.GetTime(); } @@ -301,7 +262,7 @@ class CompareTxMemPoolEntryByEntryTime class CompareTxMemPoolEntryByAncestorFee { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) { double aFees = a.GetModFeesWithAncestors(); double aSize = a.GetSizeWithAncestors(); @@ -360,6 +321,20 @@ enum class MemPoolRemovalReason { REPLACED //! Removed for replacement }; +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -431,14 +406,6 @@ enum class MemPoolRemovalReason { * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * - * Adding transactions from a disconnected block can be very time consuming, - * because we don't have a way to limit the number of in-mempool descendants. - * To bound CPU processing, we limit the amount of work we're willing to do - * to properly update the descendant information for a tx being added from - * a disconnected block. If we would exceed the limit, then we instead mark - * the entry as "dirty", and set the feerate for sorting purposes to be equal - * the feerate of the transaction without any descendants. - * */ class CTxMemPool { @@ -525,12 +492,11 @@ class CTxMemPool public: indirectmap mapNextTx; - std::map > mapDeltas; + std::map mapDeltas; /** Create a new CTxMemPool. */ - CTxMemPool(const CFeeRate& _minReasonableRelayFee); - ~CTxMemPool(); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -567,8 +533,8 @@ class CTxMemPool bool HasNoInputsOf(const CTransaction& tx) const; /** Affect CreateNewBlock prioritisation of transactions */ - void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta); - void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const; + void PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta); + void ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: @@ -585,12 +551,12 @@ class CTxMemPool * new mempool entries may have children in the mempool (which is generally * not the case when otherwise adding transactions). * UpdateTransactionsFromBlock() will find child transactions and update the - * descendant state for each transaction in hashesToUpdate (excluding any - * child transactions present in hashesToUpdate, which are already accounted - * for). Note: hashesToUpdate should be the set of transactions from the + * descendant state for each transaction in vHashesToUpdate (excluding any + * child transactions present in vHashesToUpdate, which are already accounted + * for). Note: vHashesToUpdate should be the set of transactions from the * disconnected block that have been accepted back into the mempool. */ - void UpdateTransactionsFromBlock(const std::vector &hashesToUpdate); + void UpdateTransactionsFromBlock(const std::vector &vHashesToUpdate); /** Try to calculate all in-mempool ancestors of entry. * (these are all calculated including the tx itself) @@ -618,10 +584,10 @@ class CTxMemPool CFeeRate GetMinFee(size_t sizelimit) const; /** Remove transactions from the mempool until its dynamic size is <= sizelimit. - * pvNoSpendsRemaining, if set, will be populated with the list of transactions + * pvNoSpendsRemaining, if set, will be populated with the list of outpoints * which are not in mempool which no longer have any spends in this mempool. */ - void TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining=NULL); + void TrimToSize(size_t sizelimit, std::vector* pvNoSpendsRemaining=NULL); /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); @@ -647,32 +613,17 @@ class CTxMemPool return (mapTx.count(hash) != 0); } + bool exists(const COutPoint& outpoint) const + { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); + } + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector infoAll() const; - /** Estimate fee rate needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate fee rate needed to get into the next nBlocks */ - CFeeRate estimateFee(int nBlocks) const; - - /** Estimate priority needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - double estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate priority needed to get into the next nBlocks */ - double estimatePriority(int nBlocks) const; - - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile& fileout) const; - bool ReadFeeEstimates(CAutoFile& filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal NotifyEntryAdded; @@ -717,7 +668,7 @@ class CTxMemPool void removeUnchecked(txiter entry, MemPoolRemovalReason reason = MemPoolRemovalReason::UNKNOWN); }; -/** +/** * CCoinsView that brings transactions from a memorypool into view. * It does not check for spendings by memory pool transactions. */ @@ -728,23 +679,8 @@ class CCoinsViewMemPool : public CCoinsViewBacked public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; -}; - -// We want to sort transactions by coin age priority -typedef std::pair TxCoinAgePriority; - -struct TxCoinAgePriorityCompare -{ - bool operator()(const TxCoinAgePriority& a, const TxCoinAgePriority& b) - { - if (a.first == b.first) - return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); //Reverse order to make sort less than - return a.first < b.first; - } + bool GetCoins(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoins(const COutPoint &outpoint) const; }; #endif // BITCOIN_TXMEMPOOL_H - - diff --git a/src/validation.cpp b/src/validation.cpp index 8709b956626..7aaed05f63e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -434,15 +434,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoins(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -582,9 +582,9 @@ void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { if (expired != 0) LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); - std::vector vNoSpendsRemaining; + std::vector vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining) pcoinsTip->Uncache(removed); } @@ -611,7 +611,7 @@ static bool IsCurrentForFeeEstimation() bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& vHashTxnToUncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -704,30 +704,30 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C view.SetBackend(viewMemPool); // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + for (size_t out = 0; out < tx.vout.size(); out++) { + COutPoint outpoint(hash, out); + bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(outpoint); + if (view.HaveCoins(outpoint)) { + if (!fHadTxInCache) { + vHashTxnToUncache.push_back(outpoint); + } + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } } // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) + if (!pcoinsTip->HaveCoinsInCache(txin.prevout)) { + vHashTxnToUncache.push_back(txin.prevout); + } + if (!view.HaveCoins(txin.prevout)) { + if (pfMissingInputs) { *pfMissingInputs = true; + } return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - // Bring the best block into scope view.GetBestBlock(); @@ -1085,10 +1085,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector vHashTxToUncache; + std::vector vHashTxToUncache; bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, vHashTxToUncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -2162,12 +2162,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) @@ -2248,7 +2248,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), From 1c9a91fa7bf3e7121b888b11d922b60f836c3390 Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Sat, 13 Jun 2026 01:29:38 +0000 Subject: [PATCH 22/32] Backport Session 2 per-txout migration commits with Dogecoin compatibility fixes --- src/rest.cpp | 29 +- src/rpc/blockchain.cpp | 827 +++++++++++++++++++++++++++------------ src/test/coins_tests.cpp | 4 +- src/txdb.cpp | 6 +- src/txmempool.cpp | 135 +++++-- src/txmempool.h | 88 ++++- src/validation.cpp | 32 +- 7 files changed, 816 insertions(+), 305 deletions(-) diff --git a/src/rest.cpp b/src/rest.cpp index 16c8c4f5293..1abd8fbb762 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -60,6 +60,12 @@ struct CCoin { } }; +extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails); +extern UniValue mempoolInfoToJSON(); +extern UniValue mempoolToJSON(bool fVerbose); +extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); +extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); + static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); @@ -158,8 +164,9 @@ static bool rest_headers(HTTPRequest* req, } CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION); + const CChainParams& chainparams = Params(); BOOST_FOREACH(const CBlockIndex *pindex, headers) { - ssHeader << pindex->GetBlockHeader(); + ssHeader << pindex->GetBlockHeader(chainparams.GetConsensus(pindex->nHeight)); } switch (rf) { @@ -219,7 +226,7 @@ static bool rest_block(HTTPRequest* req, if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus(pblockindex->nHeight))) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -360,7 +367,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) CTransactionRef tx; uint256 hashBlock = uint256(); - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) + if (!GetTransaction(hash, tx, Params().GetConsensus(0), hashBlock, true)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); @@ -553,23 +560,23 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) // pack in some essentials // use more or less the same output as mentioned in Bip64 - objGetUTXOResponse.push_back(Pair("chainHeight", chainActive.Height())); - objGetUTXOResponse.push_back(Pair("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex())); - objGetUTXOResponse.push_back(Pair("bitmap", bitmapStringRepresentation)); + objGetUTXOResponse.pushKV("chainHeight", chainActive.Height()); + objGetUTXOResponse.pushKV("chaintipHash", chainActive.Tip()->GetBlockHash().GetHex()); + objGetUTXOResponse.pushKV("bitmap", bitmapStringRepresentation); UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.push_back(Pair("height", (int32_t)coin.nHeight)); - utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); + utxo.pushKV("height", (int32_t)coin.nHeight); + utxo.pushKV("value", ValueFromAmount(coin.out.nValue)); // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); - utxo.push_back(Pair("scriptPubKey", o)); + ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + utxo.pushKV("scriptPubKey", o); utxos.push_back(utxo); } - objGetUTXOResponse.push_back(Pair("utxos", utxos)); + objGetUTXOResponse.pushKV("utxos", utxos); // return json string std::string strJSON = objGetUTXOResponse.write() + "\n"; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 77f1dcb21f7..a8a9310354d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,6 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2022-2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,15 +12,16 @@ #include "checkpoints.h" #include "coins.h" #include "consensus/validation.h" -#include "validation.h" #include "core_io.h" -#include "policy/feerate.h" +#include "dogecoin.h" +#include "validation.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" #include "streams.h" #include "sync.h" #include "txmempool.h" +#include "undo.h" #include "util.h" #include "utilstrencodings.h" #include "hash.h" @@ -28,10 +30,12 @@ #include +#include #include // boost::thread::interrupt #include #include +using namespace std; struct CUpdatedBlock { @@ -44,12 +48,15 @@ static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); +void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); double GetDifficulty(const CBlockIndex* blockindex) { - if (blockindex == NULL) + // Floating point number that is a multiple of the minimum difficulty, + // minimum difficulty = 1.0. + if (blockindex == nullptr) { - if (chainActive.Tip() == NULL) + if (chainActive.Tip() == nullptr) return 1.0; else blockindex = chainActive.Tip(); @@ -74,82 +81,121 @@ double GetDifficulty(const CBlockIndex* blockindex) return dDiff; } +UniValue AuxpowToJSON(const CAuxPow& auxpow) +{ + UniValue result(UniValue::VOBJ); + + { + UniValue tx(UniValue::VOBJ); + tx.pushKV("hex", EncodeHexTx(auxpow)); + TxToJSON(auxpow, auxpow.parentBlock.GetHash(), tx); + result.pushKV("tx", tx); + } + + result.pushKV("index", auxpow.nIndex); + result.pushKV("chainindex", auxpow.nChainIndex); + + { + UniValue branch(UniValue::VARR); + BOOST_FOREACH(const uint256& node, auxpow.vMerkleBranch) + branch.push_back(node.GetHex()); + result.pushKV("merklebranch", branch); + } + + { + UniValue branch(UniValue::VARR); + BOOST_FOREACH(const uint256& node, auxpow.vChainMerkleBranch) + branch.push_back(node.GetHex()); + result.pushKV("chainmerklebranch", branch); + } + + CDataStream ssParent(SER_NETWORK, PROTOCOL_VERSION); + ssParent << auxpow.parentBlock; + const std::string strHex = HexStr(ssParent.begin(), ssParent.end()); + result.pushKV("parentblock", strHex); + + return result; +} + UniValue blockheaderToJSON(const CBlockIndex* blockindex) { UniValue result(UniValue::VOBJ); - result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); + result.pushKV("hash", blockindex->GetBlockHash().GetHex()); int confirmations = -1; // Only report confirmations if the block is on the main chain if (chainActive.Contains(blockindex)) confirmations = chainActive.Height() - blockindex->nHeight + 1; - result.push_back(Pair("confirmations", confirmations)); - result.push_back(Pair("height", blockindex->nHeight)); - result.push_back(Pair("version", blockindex->nVersion)); - result.push_back(Pair("versionHex", strprintf("%08x", blockindex->nVersion))); - result.push_back(Pair("merkleroot", blockindex->hashMerkleRoot.GetHex())); - result.push_back(Pair("time", (int64_t)blockindex->nTime)); - result.push_back(Pair("mediantime", (int64_t)blockindex->GetMedianTimePast())); - result.push_back(Pair("nonce", (uint64_t)blockindex->nNonce)); - result.push_back(Pair("bits", strprintf("%08x", blockindex->nBits))); - result.push_back(Pair("difficulty", GetDifficulty(blockindex))); - result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex())); + result.pushKV("confirmations", confirmations); + result.pushKV("height", blockindex->nHeight); + result.pushKV("version", blockindex->nVersion); + result.pushKV("versionHex", strprintf("%08x", blockindex->nVersion)); + result.pushKV("merkleroot", blockindex->hashMerkleRoot.GetHex()); + result.pushKV("time", (int64_t)blockindex->nTime); + result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); + result.pushKV("nonce", (uint64_t)blockindex->nNonce); + result.pushKV("bits", strprintf("%08x", blockindex->nBits)); + result.pushKV("difficulty", GetDifficulty(blockindex)); + result.pushKV("chainwork", blockindex->nChainWork.GetHex()); if (blockindex->pprev) - result.push_back(Pair("previousblockhash", blockindex->pprev->GetBlockHash().GetHex())); + result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); CBlockIndex *pnext = chainActive.Next(blockindex); if (pnext) - result.push_back(Pair("nextblockhash", pnext->GetBlockHash().GetHex())); + result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false) { UniValue result(UniValue::VOBJ); - result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); + result.pushKV("hash", blockindex->GetBlockHash().GetHex()); int confirmations = -1; // Only report confirmations if the block is on the main chain if (chainActive.Contains(blockindex)) confirmations = chainActive.Height() - blockindex->nHeight + 1; - result.push_back(Pair("confirmations", confirmations)); - result.push_back(Pair("strippedsize", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS))); - result.push_back(Pair("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION))); - result.push_back(Pair("weight", (int)::GetBlockWeight(block))); - result.push_back(Pair("height", blockindex->nHeight)); - result.push_back(Pair("version", block.nVersion)); - result.push_back(Pair("versionHex", strprintf("%08x", block.nVersion))); - result.push_back(Pair("merkleroot", block.hashMerkleRoot.GetHex())); + result.pushKV("confirmations", confirmations); + result.pushKV("strippedsize", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); + result.pushKV("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION)); + result.pushKV("weight", (int)::GetBlockWeight(block)); + result.pushKV("height", blockindex->nHeight); + result.pushKV("version", block.nVersion); + result.pushKV("versionHex", strprintf("%08x", block.nVersion)); + result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); UniValue txs(UniValue::VARR); for(const auto& tx : block.vtx) { if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx); + TxToJSON(*tx, uint256(), objTx); txs.push_back(objTx); } else txs.push_back(tx->GetHash().GetHex()); } - result.push_back(Pair("tx", txs)); - result.push_back(Pair("time", block.GetBlockTime())); - result.push_back(Pair("mediantime", (int64_t)blockindex->GetMedianTimePast())); - result.push_back(Pair("nonce", (uint64_t)block.nNonce)); - result.push_back(Pair("bits", strprintf("%08x", block.nBits))); - result.push_back(Pair("difficulty", GetDifficulty(blockindex))); - result.push_back(Pair("chainwork", blockindex->nChainWork.GetHex())); + result.pushKV("tx", txs); + result.pushKV("time", block.GetBlockTime()); + result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); + result.pushKV("nonce", (uint64_t)block.nNonce); + result.pushKV("bits", strprintf("%08x", block.nBits)); + result.pushKV("difficulty", GetDifficulty(blockindex)); + result.pushKV("chainwork", blockindex->nChainWork.GetHex()); + + if (block.auxpow) + result.pushKV("auxpow", AuxpowToJSON(*block.auxpow)); if (blockindex->pprev) - result.push_back(Pair("previousblockhash", blockindex->pprev->GetBlockHash().GetHex())); + result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); CBlockIndex *pnext = chainActive.Next(blockindex); if (pnext) - result.push_back(Pair("nextblockhash", pnext->GetBlockHash().GetHex())); + result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); return result; } UniValue getblockcount(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "getblockcount\n" "\nReturns the number of blocks in the longest blockchain.\n" "\nResult:\n" @@ -166,7 +212,7 @@ UniValue getblockcount(const JSONRPCRequest& request) UniValue getbestblockhash(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "getbestblockhash\n" "\nReturns the hash of the best (tip) block in the longest blockchain.\n" "\nResult:\n" @@ -187,13 +233,13 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } - cond_blockchange.notify_all(); + cond_blockchange.notify_all(); } UniValue waitfornewblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw std::runtime_error( + throw runtime_error( "waitfornewblock (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" @@ -223,15 +269,15 @@ UniValue waitfornewblock(const JSONRPCRequest& request) block = latestblock; } UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("hash", block.hash.GetHex())); - ret.push_back(Pair("height", block.height)); + ret.pushKV("hash", block.hash.GetHex()); + ret.pushKV("height", block.height); return ret; } UniValue waitforblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( + throw runtime_error( "waitforblock (timeout)\n" "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n" @@ -265,15 +311,15 @@ UniValue waitforblock(const JSONRPCRequest& request) } UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("hash", block.hash.GetHex())); - ret.push_back(Pair("height", block.height)); + ret.pushKV("hash", block.hash.GetHex()); + ret.pushKV("height", block.height); return ret; } UniValue waitforblockheight(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( + throw runtime_error( "waitforblockheight (timeout)\n" "\nWaits for (at least) block height and returns the height and hash\n" "of the current tip.\n" @@ -307,15 +353,15 @@ UniValue waitforblockheight(const JSONRPCRequest& request) block = latestblock; } UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("hash", block.hash.GetHex())); - ret.push_back(Pair("height", block.height)); + ret.pushKV("hash", block.hash.GetHex()); + ret.pushKV("height", block.height); return ret; } UniValue getdifficulty(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "getdifficulty\n" "\nReturns the proof-of-work difficulty as a multiple of the minimum difficulty.\n" "\nResult:\n" @@ -336,6 +382,8 @@ std::string EntryDescriptionString() " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" " \"height\" : n, (numeric) block height when transaction entered pool\n" + " \"startingpriority\" : n, (numeric) DEPRECATED. Priority when transaction entered pool\n" + " \"currentpriority\" : n, (numeric) DEPRECATED. Transaction priority now\n" " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n" " \"descendantsize\" : n, (numeric) virtual transaction size of in-mempool descendants (including this one)\n" " \"descendantfees\" : n, (numeric) modified fees (see above) of in-mempool descendants (including this one)\n" @@ -351,19 +399,21 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) { AssertLockHeld(mempool.cs); - info.push_back(Pair("size", (int)e.GetTxSize())); - info.push_back(Pair("fee", ValueFromAmount(e.GetFee()))); - info.push_back(Pair("modifiedfee", ValueFromAmount(e.GetModifiedFee()))); - info.push_back(Pair("time", e.GetTime())); - info.push_back(Pair("height", (int)e.GetHeight())); - info.push_back(Pair("descendantcount", e.GetCountWithDescendants())); - info.push_back(Pair("descendantsize", e.GetSizeWithDescendants())); - info.push_back(Pair("descendantfees", e.GetModFeesWithDescendants())); - info.push_back(Pair("ancestorcount", e.GetCountWithAncestors())); - info.push_back(Pair("ancestorsize", e.GetSizeWithAncestors())); - info.push_back(Pair("ancestorfees", e.GetModFeesWithAncestors())); + info.pushKV("size", (int)e.GetTxSize()); + info.pushKV("fee", ValueFromAmount(e.GetFee())); + info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); + info.pushKV("time", e.GetTime()); + info.pushKV("height", (int)e.GetHeight()); + info.pushKV("startingpriority", e.GetPriority(e.GetHeight())); + info.pushKV("currentpriority", e.GetPriority(chainActive.Height())); + info.pushKV("descendantcount", e.GetCountWithDescendants()); + info.pushKV("descendantsize", e.GetSizeWithDescendants()); + info.pushKV("descendantfees", e.GetModFeesWithDescendants()); + info.pushKV("ancestorcount", e.GetCountWithAncestors()); + info.pushKV("ancestorsize", e.GetSizeWithAncestors()); + info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); const CTransaction& tx = e.GetTx(); - std::set setDepends; + set setDepends; BOOST_FOREACH(const CTxIn& txin, tx.vin) { if (mempool.exists(txin.prevout.hash)) @@ -371,15 +421,15 @@ void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) } UniValue depends(UniValue::VARR); - BOOST_FOREACH(const std::string& dep, setDepends) + BOOST_FOREACH(const string& dep, setDepends) { depends.push_back(dep); } - info.push_back(Pair("depends", depends)); + info.pushKV("depends", depends); } -UniValue mempoolToJSON(bool fVerbose) +UniValue mempoolToJSON(bool fVerbose = false) { if (fVerbose) { @@ -390,13 +440,13 @@ UniValue mempoolToJSON(bool fVerbose) const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(info, e); - o.push_back(Pair(hash.ToString(), info)); + o.pushKV(hash.ToString(), info); } return o; } else { - std::vector vtxid; + vector vtxid; mempool.queryHashes(vtxid); UniValue a(UniValue::VARR); @@ -410,7 +460,7 @@ UniValue mempoolToJSON(bool fVerbose) UniValue getrawmempool(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 1) - throw std::runtime_error( + throw runtime_error( "getrawmempool ( verbose )\n" "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" @@ -442,7 +492,7 @@ UniValue getrawmempool(const JSONRPCRequest& request) UniValue getmempoolancestors(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( + throw runtime_error( "getmempoolancestors txid (verbose)\n" "\nIf txid is in the mempool, returns all in-mempool ancestors.\n" "\nArguments:\n" @@ -497,7 +547,7 @@ UniValue getmempoolancestors(const JSONRPCRequest& request) const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(info, e); - o.push_back(Pair(_hash.ToString(), info)); + o.pushKV(_hash.ToString(), info); } return o; } @@ -506,7 +556,7 @@ UniValue getmempoolancestors(const JSONRPCRequest& request) UniValue getmempooldescendants(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( + throw runtime_error( "getmempooldescendants txid (verbose)\n" "\nIf txid is in the mempool, returns all in-mempool descendants.\n" "\nArguments:\n" @@ -561,7 +611,7 @@ UniValue getmempooldescendants(const JSONRPCRequest& request) const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); entryToJSON(info, e); - o.push_back(Pair(_hash.ToString(), info)); + o.pushKV(_hash.ToString(), info); } return o; } @@ -570,7 +620,7 @@ UniValue getmempooldescendants(const JSONRPCRequest& request) UniValue getmempoolentry(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) { - throw std::runtime_error( + throw runtime_error( "getmempoolentry txid\n" "\nReturns mempool data for given transaction\n" "\nArguments:\n" @@ -603,7 +653,7 @@ UniValue getmempoolentry(const JSONRPCRequest& request) UniValue getblockhash(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( + throw runtime_error( "getblockhash height\n" "\nReturns hash of block in best-block-chain at height provided.\n" "\nArguments:\n" @@ -628,7 +678,7 @@ UniValue getblockhash(const JSONRPCRequest& request) UniValue getblockheader(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( + throw runtime_error( "getblockheader \"hash\" ( verbose )\n" "\nIf verbose is false, returns a string that is serialized, hex-encoded data for blockheader 'hash'.\n" "If verbose is true, returns an Object with information about blockheader .\n" @@ -676,7 +726,7 @@ UniValue getblockheader(const JSONRPCRequest& request) if (!fVerbose) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); - ssBlock << pblockindex->GetBlockHeader(); + ssBlock << pblockindex->GetBlockHeader(Params().GetConsensus(pblockindex->nHeight)); std::string strHex = HexStr(ssBlock.begin(), ssBlock.end()); return strHex; } @@ -684,10 +734,47 @@ UniValue getblockheader(const JSONRPCRequest& request) return blockheaderToJSON(pblockindex); } +static CBlock GetBlockChecked(const CBlockIndex* pblockindex) +{ + CBlock block; + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); + } + + if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus(pblockindex->nHeight))) { + // Block not found on disk. This could be because we have the block + // header in our index but don't have the block (for example if a + // non-whitelisted node sends us an unrequested long chain of valid + // blocks, we add the headers to our index, but don't accept the + // block). + throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); + } + + return block; +} + +static CBlockUndo GetUndoChecked(const CBlockIndex* pblockindex) +{ + CBlockUndo blockUndo; + + // The Genesis block does not have undo data + if (pblockindex->nHeight == 0) return blockUndo; + + if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { + throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); + } + + if (!UndoReadFromDisk(blockUndo, pblockindex->GetUndoPos(), pblockindex->pprev->GetBlockHash())) { + throw JSONRPCError(RPC_MISC_ERROR, "Can't read undo data from disk"); + } + + return blockUndo; +} + UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) - throw std::runtime_error( + throw runtime_error( "getblock \"blockhash\" ( verbosity ) \n" "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block .\n" @@ -734,35 +821,30 @@ UniValue getblock(const JSONRPCRequest& request) + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") ); - LOCK(cs_main); - std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int verbosity = 1; if (request.params.size() > 1) { - if(request.params[1].isNum()) + if(request.params[1].isNum()) { verbosity = request.params[1].get_int(); - else + } else { verbosity = request.params[1].get_bool() ? 1 : 0; + } } - if (mapBlockIndex.count(hash) == 0) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - CBlock block; - CBlockIndex* pblockindex = mapBlockIndex[hash]; + const CBlockIndex* pblockindex; + { + LOCK(cs_main); - if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); + if (mapBlockIndex.count(hash) == 0) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) - // Block not found on disk. This could be because we have the block - // header in our index but don't have the block (for example if a - // non-whitelisted node sends us an unrequested long chain of valid - // blocks, we add the headers to our index, but don't accept the - // block). - throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); + pblockindex = mapBlockIndex[hash]; + + block = GetBlockChecked(pblockindex); + } if (verbosity <= 0) { @@ -783,9 +865,9 @@ struct CCoinsStats uint64_t nTransactionOutputs; uint256 hashSerialized; uint64_t nDiskSize; - CAmount nTotalAmount; + arith_uint256 nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nDiskSize(0), nTotalAmount(0) {} }; static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) @@ -845,7 +927,7 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( + throw runtime_error( "pruneblockchain\n" "\nArguments:\n" "1. \"height\" (numeric, required) The block height to prune up to. May be set to a discrete height, or a unix timestamp\n" @@ -869,7 +951,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW); + CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - 7200); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -883,7 +965,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) else if (height > chainHeight) throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height."); else if (height > chainHeight - MIN_BLOCKS_TO_KEEP) { - LogPrint(BCLog::RPC, "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); + LogPrint("rpc", "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks."); height = chainHeight - MIN_BLOCKS_TO_KEEP; } @@ -894,7 +976,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) UniValue gettxoutsetinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "gettxoutsetinfo\n" "\nReturns statistics about the unspent transaction output set.\n" "Note this call may take some time.\n" @@ -904,7 +986,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"hash_serialized_2\": \"hash\", (string) The serialized hash\n" " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" @@ -918,13 +1000,13 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); if (GetUTXOStats(pcoinsTip, stats)) { - ret.push_back(Pair("height", (int64_t)stats.nHeight)); - ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); - ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); - ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); - ret.push_back(Pair("hash_serialized_2", stats.hashSerialized.GetHex())); - ret.push_back(Pair("disk_size", stats.nDiskSize)); - ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); + ret.pushKV("height", (int64_t)stats.nHeight); + ret.pushKV("bestblock", stats.hashBlock.GetHex()); + ret.pushKV("transactions", (int64_t)stats.nTransactions); + ret.pushKV("txouts", (int64_t)stats.nTransactionOutputs); + ret.pushKV("hash_serialized_2", stats.hashSerialized.GetHex()); + ret.pushKV("disk_size", (int64_t)stats.nDiskSize); + ret.pushKV("total_amount", ValueFromAmount(stats.nTotalAmount)); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } @@ -934,7 +1016,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) UniValue gettxout(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) - throw std::runtime_error( + throw runtime_error( "gettxout \"txid\" n ( include_mempool )\n" "\nReturns details about an unspent transaction output.\n" "\nArguments:\n" @@ -951,12 +1033,11 @@ UniValue gettxout(const JSONRPCRequest& request) " \"hex\" : \"hex\", (string) \n" " \"reqSigs\" : n, (numeric) Number of required signatures\n" " \"type\" : \"pubkeyhash\", (string) The type, eg pubkeyhash\n" - " \"addresses\" : [ (array of string) array of bitcoin addresses\n" - " \"address\" (string) bitcoin address\n" + " \"addresses\" : [ (array of string) array of dogecoin addresses\n" + " \"address\" (string) dogecoin address\n" " ,...\n" " ]\n" " },\n" - " \"version\" : n, (numeric) The version\n" " \"coinbase\" : true|false (boolean) Coinbase or not\n" "}\n" @@ -976,37 +1057,36 @@ UniValue gettxout(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int n = request.params[1].get_int(); - COutPoint out(hash, n); bool fMempool = true; if (request.params.size() > 2) fMempool = request.params[2].get_bool(); + COutPoint out(hash, n); Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(out, coin) || mempool.isSpent(out)) { // TODO: filtering spent coins should be done by the CCoinsViewMemPool + if (!view.GetCoins(out, coin) || mempool.isSpent(out)) // TODO: this should be done by the CCoinsViewMemPool return NullUniValue; - } } else { - if (!pcoinsTip->GetCoins(out, coin)) { + if (!pcoinsTip->GetCoins(out, coin)) return NullUniValue; - } } + if (coin.IsPruned()) + return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; - ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); - if (coin.nHeight == MEMPOOL_HEIGHT) { - ret.push_back(Pair("confirmations", 0)); - } else { - ret.push_back(Pair("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1))); - } - ret.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); + ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); + if (coin.nHeight == MEMPOOL_HEIGHT) + ret.pushKV("confirmations", 0); + else + ret.pushKV("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1)); + ret.pushKV("value", ValueFromAmount(coin.out.nValue)); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); - ret.push_back(Pair("scriptPubKey", o)); - ret.push_back(Pair("coinbase", (bool)coin.fCoinBase)); + ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + ret.pushKV("scriptPubKey", o); + ret.pushKV("coinbase", (bool)coin.fCoinBase); return ret; } @@ -1016,7 +1096,7 @@ UniValue verifychain(const JSONRPCRequest& request) int nCheckLevel = GetArg("-checklevel", DEFAULT_CHECKLEVEL); int nCheckDepth = GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); if (request.fHelp || request.params.size() > 2) - throw std::runtime_error( + throw runtime_error( "verifychain ( checklevel nblocks )\n" "\nVerifies blockchain database.\n" "\nArguments:\n" @@ -1033,8 +1113,13 @@ UniValue verifychain(const JSONRPCRequest& request) if (request.params.size() > 0) nCheckLevel = request.params[0].get_int(); + if (nCheckLevel < 0 || nCheckLevel > 4) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: checklevel must be >= 0 and <= 4"); + if (request.params.size() > 1) nCheckDepth = request.params[1].get_int(); + if (nCheckDepth < 0) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: nblocks must be >= 0"); return CVerifyDB().VerifyDB(Params(), pcoinsTip, nCheckLevel, nCheckDepth); } @@ -1056,16 +1141,16 @@ static UniValue SoftForkMajorityDesc(int version, CBlockIndex* pindex, const Con activated = pindex->nHeight >= consensusParams.BIP65Height; break; } - rv.push_back(Pair("status", activated)); + rv.pushKV("status", activated); return rv; } static UniValue SoftForkDesc(const std::string &name, int version, CBlockIndex* pindex, const Consensus::Params& consensusParams) { UniValue rv(UniValue::VOBJ); - rv.push_back(Pair("id", name)); - rv.push_back(Pair("version", version)); - rv.push_back(Pair("reject", SoftForkMajorityDesc(version, pindex, consensusParams))); + rv.pushKV("id", name); + rv.pushKV("version", version); + rv.pushKV("reject", SoftForkMajorityDesc(version, pindex, consensusParams)); return rv; } @@ -1074,30 +1159,19 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse UniValue rv(UniValue::VOBJ); const ThresholdState thresholdState = VersionBitsTipState(consensusParams, id); switch (thresholdState) { - case THRESHOLD_DEFINED: rv.push_back(Pair("status", "defined")); break; - case THRESHOLD_STARTED: rv.push_back(Pair("status", "started")); break; - case THRESHOLD_LOCKED_IN: rv.push_back(Pair("status", "locked_in")); break; - case THRESHOLD_ACTIVE: rv.push_back(Pair("status", "active")); break; - case THRESHOLD_FAILED: rv.push_back(Pair("status", "failed")); break; + case THRESHOLD_DEFINED: rv.pushKV("status", "defined"); break; + case THRESHOLD_STARTED: rv.pushKV("status", "started"); break; + case THRESHOLD_LOCKED_IN: rv.pushKV("status", "locked_in"); break; + case THRESHOLD_ACTIVE: rv.pushKV("status", "active"); break; + case THRESHOLD_FAILED: rv.pushKV("status", "failed"); break; } if (THRESHOLD_STARTED == thresholdState) { - rv.push_back(Pair("bit", consensusParams.vDeployments[id].bit)); - } - rv.push_back(Pair("startTime", consensusParams.vDeployments[id].nStartTime)); - rv.push_back(Pair("timeout", consensusParams.vDeployments[id].nTimeout)); - rv.push_back(Pair("since", VersionBitsTipStateSinceHeight(consensusParams, id))); - if (THRESHOLD_STARTED == thresholdState) - { - UniValue statsUV(UniValue::VOBJ); - BIP9Stats statsStruct = VersionBitsTipStatistics(consensusParams, id); - statsUV.push_back(Pair("period", statsStruct.period)); - statsUV.push_back(Pair("threshold", statsStruct.threshold)); - statsUV.push_back(Pair("elapsed", statsStruct.elapsed)); - statsUV.push_back(Pair("count", statsStruct.count)); - statsUV.push_back(Pair("possible", statsStruct.possible)); - rv.push_back(Pair("statistics", statsUV)); + rv.pushKV("bit", consensusParams.vDeployments[id].bit); } + rv.pushKV("startTime", consensusParams.vDeployments[id].nStartTime); + rv.pushKV("timeout", consensusParams.vDeployments[id].nTimeout); + rv.pushKV("since", VersionBitsTipStateSinceHeight(consensusParams, id)); return rv; } @@ -1107,13 +1181,13 @@ void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const std::string &name, // A timeout value of 0 guarantees a softfork will never be activated. // This is used when softfork codes are merged without specifying the deployment schedule. if (consensusParams.vDeployments[id].nTimeout > 0) - bip9_softforks.push_back(Pair(name, BIP9SoftForkDesc(consensusParams, id))); + bip9_softforks.pushKV(name, BIP9SoftForkDesc(consensusParams, id)); } UniValue getblockchaininfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "getblockchaininfo\n" "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" @@ -1125,9 +1199,13 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" + " \"initialblockdownload\": xxxx, (bool) (debug information) estimate of whether this node is in Initial Block Download mode.\n" " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" + " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" - " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored\n" + " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" + " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" + " \"prune_target_size\": xxxxxx, (numeric) the target size used by pruning (only present if automatic pruning is enabled)\n" " \"softforks\": [ (array) status of softforks in progress\n" " {\n" " \"id\": \"xxxx\", (string) name of softfork\n" @@ -1143,14 +1221,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx, (numeric) height of the first block to which the status applies\n" - " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" - " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" - " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" - " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" - " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" - " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" - " }\n" + " \"since\": xx (numeric) height of the first block to which the status applies\n" " }\n" " }\n" "}\n" @@ -1162,17 +1233,35 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) LOCK(cs_main); UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("chain", Params().NetworkIDString())); - obj.push_back(Pair("blocks", (int)chainActive.Height())); - obj.push_back(Pair("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1)); - obj.push_back(Pair("bestblockhash", chainActive.Tip()->GetBlockHash().GetHex())); - obj.push_back(Pair("difficulty", (double)GetDifficulty())); - obj.push_back(Pair("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast())); - obj.push_back(Pair("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip()))); - obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); - obj.push_back(Pair("pruned", fPruneMode)); - - const Consensus::Params& consensusParams = Params().GetConsensus(); + obj.pushKV("chain", Params().NetworkIDString()); + obj.pushKV("blocks", (int)chainActive.Height()); + obj.pushKV("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1); + obj.pushKV("bestblockhash", chainActive.Tip()->GetBlockHash().GetHex()); + obj.pushKV("difficulty", (double)GetDifficulty()); + obj.pushKV("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast()); + obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip())); + obj.pushKV("initialblockdownload", IsInitialBlockDownload()); + obj.pushKV("chainwork", chainActive.Tip()->nChainWork.GetHex()); + obj.pushKV("size_on_disk", CalculateCurrentUsage()); + obj.pushKV("pruned", fPruneMode); + if (fPruneMode) { + CBlockIndex* block = chainActive.Tip(); + assert(block); + while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { + block = block->pprev; + } + + obj.pushKV("pruneheight", block->nHeight); + + // if 0, execution bypasses the whole if block. + bool automatic_pruning = (GetArg("-prune", 0) != 1); + obj.pushKV("automatic_pruning", automatic_pruning); + if (automatic_pruning) { + obj.pushKV("prune_target_size", nPruneTarget); + } + } + + const Consensus::Params& consensusParams = Params().GetConsensus(0); CBlockIndex* tip = chainActive.Tip(); UniValue softforks(UniValue::VARR); UniValue bip9_softforks(UniValue::VOBJ); @@ -1181,17 +1270,9 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) softforks.push_back(SoftForkDesc("bip65", 4, tip, consensusParams)); BIP9SoftForkDescPushBack(bip9_softforks, "csv", consensusParams, Consensus::DEPLOYMENT_CSV); BIP9SoftForkDescPushBack(bip9_softforks, "segwit", consensusParams, Consensus::DEPLOYMENT_SEGWIT); - obj.push_back(Pair("softforks", softforks)); - obj.push_back(Pair("bip9_softforks", bip9_softforks)); - - if (fPruneMode) - { - CBlockIndex *block = chainActive.Tip(); - while (block && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) - block = block->pprev; - - obj.push_back(Pair("pruneheight", block->nHeight)); - } + obj.pushKV("softforks", softforks); + obj.pushKV("bip9_softforks", bip9_softforks); + obj.pushKV("warnings", GetWarnings("statusbar")); return obj; } @@ -1213,7 +1294,7 @@ struct CompareBlocksByHeight UniValue getchaintips(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "getchaintips\n" "Return information about all known tips in the block tree," " including the main chain as well as orphaned branches.\n" @@ -1246,7 +1327,7 @@ UniValue getchaintips(const JSONRPCRequest& request) LOCK(cs_main); /* - * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off of them. + * Idea: the set of chain tips is chainActive.tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: * - Make one pass through mapBlockIndex, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. @@ -1279,13 +1360,13 @@ UniValue getchaintips(const JSONRPCRequest& request) BOOST_FOREACH(const CBlockIndex* block, setTips) { UniValue obj(UniValue::VOBJ); - obj.push_back(Pair("height", block->nHeight)); - obj.push_back(Pair("hash", block->phashBlock->GetHex())); + obj.pushKV("height", block->nHeight); + obj.pushKV("hash", block->phashBlock->GetHex()); const int branchLen = block->nHeight - chainActive.FindFork(block)->nHeight; - obj.push_back(Pair("branchlen", branchLen)); + obj.pushKV("branchlen", branchLen); - std::string status; + string status; if (chainActive.Contains(block)) { // This block is part of the currently active chain. status = "active"; @@ -1305,7 +1386,7 @@ UniValue getchaintips(const JSONRPCRequest& request) // No clue. status = "unknown"; } - obj.push_back(Pair("status", status)); + obj.pushKV("status", status); res.push_back(obj); } @@ -1316,12 +1397,12 @@ UniValue getchaintips(const JSONRPCRequest& request) UniValue mempoolInfoToJSON() { UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("size", (int64_t) mempool.size())); - ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize())); - ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); + ret.pushKV("size", (int64_t) mempool.size()); + ret.pushKV("bytes", (int64_t) mempool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t) mempool.DynamicMemoryUsage()); size_t maxmempool = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - ret.push_back(Pair("maxmempool", (int64_t) maxmempool)); - ret.push_back(Pair("mempoolminfee", ValueFromAmount(mempool.GetMinFee(maxmempool).GetFeePerK()))); + ret.pushKV("maxmempool", (int64_t) maxmempool); + ret.pushKV("mempoolminfee", ValueFromAmount(mempool.GetMinFee(maxmempool).GetFeePerK())); return ret; } @@ -1329,7 +1410,7 @@ UniValue mempoolInfoToJSON() UniValue getmempoolinfo(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 0) - throw std::runtime_error( + throw runtime_error( "getmempoolinfo\n" "\nReturns details on the active state of the TX memory pool.\n" "\nResult:\n" @@ -1351,7 +1432,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) UniValue preciousblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( + throw runtime_error( "preciousblock \"blockhash\"\n" "\nTreats a block as if it were received before others with the same work.\n" "\nA later preciousblock call can override the effect of an earlier one.\n" @@ -1389,7 +1470,7 @@ UniValue preciousblock(const JSONRPCRequest& request) UniValue invalidateblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( + throw runtime_error( "invalidateblock \"blockhash\"\n" "\nPermanently marks a block as invalid, as if it violated a consensus rule.\n" "\nArguments:\n" @@ -1427,7 +1508,7 @@ UniValue invalidateblock(const JSONRPCRequest& request) UniValue reconsiderblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) - throw std::runtime_error( + throw runtime_error( "reconsiderblock \"blockhash\"\n" "\nRemoves invalidity status of a block and its descendants, reconsider them for activation.\n" "This can be used to undo the effects of invalidateblock.\n" @@ -1461,68 +1542,326 @@ UniValue reconsiderblock(const JSONRPCRequest& request) return NullUniValue; } -UniValue getchaintxstats(const JSONRPCRequest& request) +template +static T CalculateTruncatedMedian(std::vector& scores) { - if (request.fHelp || request.params.size() > 2) + size_t size = scores.size(); + if (size == 0) { + return 0; + } + + std::sort(scores.begin(), scores.end()); + if (size % 2 == 0) { + return (scores[size / 2 - 1] + scores[size / 2]) / 2; + } else { + return scores[size / 2]; + } +} + +void CalculatePercentilesBySize(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector>& scores, int64_t total_size) +{ + if (scores.empty()) { + return; + } + + std::sort(scores.begin(), scores.end()); + + // 10th, 25th, 50th, 75th, and 90th percentile weight units. + const double weights[NUM_GETBLOCKSTATS_PERCENTILES] = { + total_size / 10.0, total_size / 4.0, total_size / 2.0, (total_size * 3.0) / 4.0, (total_size * 9.0) / 10.0 + }; + + int64_t next_percentile_index = 0; + int64_t cumulative_weight = 0; + for (const auto& element : scores) { + cumulative_weight += element.second; + while (next_percentile_index < NUM_GETBLOCKSTATS_PERCENTILES && cumulative_weight >= weights[next_percentile_index]) { + result[next_percentile_index] = element.first; + ++next_percentile_index; + } + } + + // Fill any remaining percentiles with the last value. + for (int64_t i = next_percentile_index; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) { + result[i] = scores.back().first; + } +} + +template +static inline bool SetHasKeys(const std::set& set) {return false;} +template +static inline bool SetHasKeys(const std::set& set, const Tk& key, const Args&... args) +{ + return (set.count(key) != 0) || SetHasKeys(set, args...); +} + +// outpoint (needed for the utxo index) + nHeight + fCoinBase +static constexpr size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) + sizeof(bool); + +static UniValue getblockstats(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { throw std::runtime_error( - "getchaintxstats ( nblocks blockhash )\n" - "\nCompute statistics about the total number and rate of transactions in the chain.\n" + "getblockstats hash ( stats )\n" + "\nCompute per block statistics for a given window. All amounts are in koinus.\n" + "It won't work for some heights with pruning.\n" "\nArguments:\n" - "1. nblocks (numeric, optional) Size of the window in number of blocks (default: one month).\n" - "2. \"blockhash\" (string, optional) The hash of the block that ends the window.\n" + "1. \"hash\" (string, required) The block hash of the target block\n" + "2. \"stats\" (array, optional) Values to plot, by default all values (see result below)\n" + " [\n" + " \"height\", (string, optional) Selected statistic\n" + " \"time\", (string, optional) Selected statistic\n" + " ,...\n" + " ]\n" "\nResult:\n" - "{\n" - " \"time\": xxxxx, (numeric) The timestamp for the statistics in UNIX format.\n" - " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" - " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window.\n" + "{ (json object)\n" + " \"avgfee\": xxxxx, (numeric) Average fee in the block\n" + " \"avgfeerate\": xxxxx, (numeric) Average feerate (in koinu per byte)\n" + " \"avgtxsize\": xxxxx, (numeric) Average transaction size\n" + " \"blockhash\": xxxxx, (string) The block hash (to check for potential reorgs)\n" + " \"dustouts\": xxxxx, (numeric) Number of outputs under the dust limit (excluding coinbase and OP_RETURN)\n" + " \"feerate_percentiles\": [ (array of numeric) Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in koinu per byte)\n" + " \"10th_percentile_feerate\", (numeric) The 10th percentile feerate\n" + " \"25th_percentile_feerate\", (numeric) The 25th percentile feerate\n" + " \"50th_percentile_feerate\", (numeric) The 50th percentile feerate\n" + " \"75th_percentile_feerate\", (numeric) The 75th percentile feerate\n" + " \"90th_percentile_feerate\", (numeric) The 90th percentile feerate\n" + " ],\n" + " \"height\": xxxxx, (numeric) The height of the block\n" + " \"ins\": xxxxx, (numeric) The number of inputs (excluding coinbase)\n" + " \"maxfee\": xxxxx, (numeric) Maximum fee in the block\n" + " \"maxfeerate\": xxxxx, (numeric) Maximum feerate (in koinu per byte)\n" + " \"maxoutamount\": xxxxx, (numeric) Maximum output value (excluding coinbase and OP_RETURN)\n" + " \"maxtxsize\": xxxxx, (numeric) Maximum transaction size\n" + " \"medianfee\": xxxxx, (numeric) Truncated median fee in the block\n" + " \"mediantime\": xxxxx, (numeric) The block median time past\n" + " \"mediantxsize\": xxxxx, (numeric) Truncated median transaction size\n" + " \"minfee\": xxxxx, (numeric) Minimum fee in the block\n" + " \"minfeerate\": xxxxx, (numeric) Minimum feerate (in koinu per byte)\n" + " \"minoutamount\": xxxxx, (numeric) Minimum output value (excluding coinbase and OP_RETURN)\n" + " \"mintxsize\": xxxxx, (numeric) Minimum transaction size\n" + " \"outs\": xxxxx, (numeric) The number of outputs\n" + " \"subsidy\": xxxxx, (numeric) The block subsidy\n" + " \"time\": xxxxx, (numeric) The block time\n" + " \"total_out\": xxxxx, (numeric) Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])\n" + " \"total_size\": xxxxx, (numeric) Total size of all non-coinbase transactions\n" + " \"total_weight\": xxxxx, (numeric) Total weight of all non-coinbase transactions\n" + " \"totalfee\": xxxxx, (numeric) The fee total\n" + " \"txs\": xxxxx, (numeric) The number of transactions (including coinbase)\n" + " \"utxo_increase\": xxxxx, (numeric) The increase/decrease in the number of unspent outputs\n" + " \"utxo_size_inc\": xxxxx, (numeric) The increase/decrease in size for the utxo index (not discounting op_return and similar)\n" "}\n" "\nExamples:\n" - + HelpExampleCli("getchaintxstats", "") - + HelpExampleRpc("getchaintxstats", "2016") + + HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + + HelpExampleRpc("getblockstats", R"("00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09", ["minfeerate","avgfeerate"])") ); + } + + LOCK(cs_main); + + CBlockIndex* pindex; + const uint256 hash = ParseHashV(request.params[0], "hash"); - const CBlockIndex* pindex; - int blockcount = 30 * 24 * 60 * 60 / Params().GetConsensus().nPowTargetSpacing; // By default: 1 month + if (mapBlockIndex.count(hash) == 0) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + pindex = mapBlockIndex[hash]; - if (request.params.size() > 0 && !request.params[0].isNull()) { - blockcount = request.params[0].get_int(); + if (!chainActive.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Block is not in chain %s", Params().NetworkIDString())); } - bool havehash = request.params.size() > 1 && !request.params[1].isNull(); - uint256 hash; - if (havehash) { - hash = uint256S(request.params[1].get_str()); + assert(pindex != nullptr); + + std::set stats; + if (!request.params[1].isNull()) { + const UniValue stats_univalue = request.params[1].get_array(); + for (unsigned int i = 0; i < stats_univalue.size(); i++) { + const std::string stat = stats_univalue[i].get_str(); + stats.insert(stat); + } } - { - LOCK(cs_main); - if (havehash) { - auto it = mapBlockIndex.find(hash); - if (it == mapBlockIndex.end()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + const CBlock block = GetBlockChecked(pindex); + const CBlockUndo blockUndo = GetUndoChecked(pindex); + + const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default) + const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0; + const bool do_medianfee = do_all || stats.count("medianfee") != 0; + const bool do_feerate_percentiles = do_all || stats.count("feerate_percentiles") != 0; + const bool do_duststats = do_all || SetHasKeys(stats, "minoutamount", "maxoutamount", "dustouts"); + const bool loop_inputs = do_all || do_medianfee || do_feerate_percentiles || + SetHasKeys(stats, "utxo_size_inc", "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate", "subsidy"); + const bool loop_outputs = do_all || loop_inputs || do_duststats || stats.count("total_out"); + const bool do_calculate_size = do_all || do_mediantxsize || + SetHasKeys(stats, "total_size", "avgtxsize", "mintxsize", "maxtxsize", "avgfeerate", "avgfeerate", "feerate_percentiles", "minfeerate", "maxfeerate"); + const bool do_calculate_weight = do_all || SetHasKeys(stats, "total_weight"); + + CAmount maxfee = 0; + CAmount maxfeerate = 0; + CAmount maxoutamount = 0; + CAmount minfee = MAX_MONEY; + CAmount minfeerate = MAX_MONEY; + CAmount minoutamount = MAX_MONEY; + CAmount total_out = 0; + CAmount totalfee = 0; + int64_t dustouts = 0; + int64_t inputs = 0; + int64_t maxtxsize = 0; + int64_t mintxsize = MAX_BLOCK_SERIALIZED_SIZE; + int64_t outputs = 0; + int64_t total_size = 0; + int64_t total_weight = 0; + int64_t utxo_size_inc = 0; + std::vector fee_array; + std::vector> feerate_array; + std::vector txsize_array; + + bool witnessEnabled = IsWitnessEnabled(pindex, Params().GetConsensus(pindex->nHeight)); + txnouttype whichType; + + for (size_t i = 0; i < block.vtx.size(); ++i) { + const auto& tx = block.vtx.at(i); + outputs += tx->vout.size(); + + CAmount tx_total_out = 0; + if (loop_outputs) { + for (const CTxOut& out : tx->vout) { + tx_total_out += out.nValue; + utxo_size_inc += GetSerializeSize(out, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + if (do_duststats) { + if (tx->IsCoinBase()) { + continue; + } + + if (::IsStandard(out.scriptPubKey, whichType, witnessEnabled)) { + if (whichType == TX_NULL_DATA) { + continue; + } + } + + if (out.nValue > maxoutamount) { + maxoutamount = out.nValue; + } + if (out.nValue < minoutamount) { + minoutamount = out.nValue; + } + if (out.nValue < nDustLimit) { + dustouts += 1; + } + } } - pindex = it->second; - if (!chainActive.Contains(pindex)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); + } + + if (tx->IsCoinBase()) { + continue; + } + + inputs += tx->vin.size(); // Don't count coinbase's fake input + total_out += tx_total_out; // Don't count coinbase reward + + int64_t tx_size = 0; + if (do_calculate_size) { + + tx_size = tx->GetTotalSize(); + if (do_mediantxsize) { + txsize_array.push_back(tx_size); } - } else { - pindex = chainActive.Tip(); + maxtxsize = std::max(maxtxsize, tx_size); + mintxsize = std::min(mintxsize, tx_size); + total_size += tx_size; + } + + int64_t weight = 0; + if (do_calculate_weight) { + weight = GetTransactionWeight(*tx); + total_weight += weight; + } + + if (loop_inputs) { + CAmount tx_total_in = 0; + const auto& txundo = blockUndo.vtxundo.at(i - 1); + for (const Coin& coin: txundo.vprevout) { + const CTxOut& prevoutput = coin.out; + + tx_total_in += prevoutput.nValue; + utxo_size_inc -= GetSerializeSize(prevoutput, SER_NETWORK, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + } + + CAmount txfee = tx_total_in - tx_total_out; + assert(MoneyRange(txfee)); + if (do_medianfee) { + fee_array.push_back(txfee); + } + maxfee = std::max(maxfee, txfee); + minfee = std::min(minfee, txfee); + totalfee += txfee; + + CAmount feerate = tx_size ? txfee / tx_size : 0; // Unit: koinu/byte + if (do_feerate_percentiles) { + feerate_array.emplace_back(std::make_pair(feerate, tx_size)); + } + maxfeerate = std::max(maxfeerate, feerate); + minfeerate = std::min(minfeerate, feerate); } } - if (blockcount < 1 || blockcount >= pindex->nHeight) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 1 and the block's height"); + CAmount feerate_percentiles[NUM_GETBLOCKSTATS_PERCENTILES] = { 0 }; + CalculatePercentilesBySize(feerate_percentiles, feerate_array, total_size); + + UniValue feerates_res(UniValue::VARR); + for (int64_t i = 0; i < NUM_GETBLOCKSTATS_PERCENTILES; i++) { + feerates_res.push_back(feerate_percentiles[i]); } - const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); - int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); - int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + CAmount subsidy = 0; + for (const auto& tx: block.vtx[0]->vout) { + subsidy += tx.nValue; + } + subsidy = std::max(subsidy - totalfee, (CAmount)0); + + UniValue ret_all(UniValue::VOBJ); + ret_all.pushKV("avgfee", (block.vtx.size() > 1) ? totalfee / (block.vtx.size() - 1) : 0); + ret_all.pushKV("avgfeerate", total_size ? totalfee / total_size : 0); // Unit: koinu/byte + ret_all.pushKV("avgtxsize", (block.vtx.size() > 1) ? total_size / (block.vtx.size() - 1) : 0); + ret_all.pushKV("blockhash", pindex->GetBlockHash().GetHex()); + ret_all.pushKV("dustouts", dustouts); + ret_all.pushKV("feerate_percentiles", feerates_res); + ret_all.pushKV("height", (int64_t)pindex->nHeight); + ret_all.pushKV("ins", inputs); + ret_all.pushKV("maxfee", maxfee); + ret_all.pushKV("maxfeerate", maxfeerate); + ret_all.pushKV("maxoutamount", maxoutamount); + ret_all.pushKV("maxtxsize", maxtxsize); + ret_all.pushKV("medianfee", CalculateTruncatedMedian(fee_array)); + ret_all.pushKV("mediantime", pindex->GetMedianTimePast()); + ret_all.pushKV("mediantxsize", CalculateTruncatedMedian(txsize_array)); + ret_all.pushKV("minfee", (minfee == MAX_MONEY) ? 0 : minfee); + ret_all.pushKV("minfeerate", (minfeerate == MAX_MONEY) ? 0 : minfeerate); + ret_all.pushKV("minoutamount", (minoutamount == MAX_MONEY) ? 0 : minoutamount); + ret_all.pushKV("mintxsize", mintxsize == MAX_BLOCK_SERIALIZED_SIZE ? 0 : mintxsize); + ret_all.pushKV("outs", outputs); + ret_all.pushKV("subsidy", subsidy); + ret_all.pushKV("time", pindex->GetBlockTime()); + ret_all.pushKV("total_out", total_out); + ret_all.pushKV("total_size", total_size); + ret_all.pushKV("total_weight", total_weight); + ret_all.pushKV("totalfee", totalfee); + ret_all.pushKV("txs", (int64_t)block.vtx.size()); + ret_all.pushKV("utxo_increase", outputs - inputs); + ret_all.pushKV("utxo_size_inc", utxo_size_inc); + + if (do_all) { + return ret_all; + } UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("time", (int64_t)pindex->nTime)); - ret.push_back(Pair("txcount", (int64_t)pindex->nChainTx)); - ret.push_back(Pair("txrate", ((double)nTxDiff) / nTimeDiff)); - + for (const std::string& stat : stats) { + const UniValue& value = ret_all[stat]; + if (value.isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid selected statistic %s", stat)); + } + ret.pushKV(stat, value); + } return ret; } @@ -1530,10 +1869,10 @@ static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} }, - { "blockchain", "getchaintxstats", &getchaintxstats, true, {"nblocks", "blockhash"} }, + { "blockchain", "getblockstats", &getblockstats, true, {"hash", "stats"} }, { "blockchain", "getbestblockhash", &getbestblockhash, true, {} }, { "blockchain", "getblockcount", &getblockcount, true, {} }, - { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity|verbose"} }, + { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity"} }, { "blockchain", "getblockhash", &getblockhash, true, {"height"} }, { "blockchain", "getblockheader", &getblockheader, true, {"blockhash","verbose"} }, { "blockchain", "getchaintips", &getchaintips, true, {} }, diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 8d519c644b9..9fdf60cd787 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,7 +8,7 @@ #include "undo.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" -#include "test/test_random.h" +#include "random.h" #include "validation.h" #include "consensus/validation.h" @@ -17,6 +17,8 @@ #include +static uint32_t insecure_rand() { return InsecureRand32(); } + int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); diff --git a/src/txdb.cpp b/src/txdb.cpp index 505ceb6089d..02f9dbd86ac 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -88,9 +88,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - bool ret = db.WriteBatch(batch); - LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); - return ret; + return db.WriteBatch(batch); } size_t CCoinsViewDB::EstimateSize() const @@ -238,7 +236,7 @@ bool CBlockTreeDB::LoadBlockIndexGuts(std::functionnStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; - if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, Params().GetConsensus())) + if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, Params().GetConsensus(pindexNew->nHeight))) return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString()); pcursor->Next(); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 328573da498..7797783aec5 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -6,24 +6,26 @@ #include "txmempool.h" #include "consensus/consensus.h" -#include "consensus/tx_verify.h" #include "consensus/validation.h" +#include "chainparams.h" #include "validation.h" #include "policy/policy.h" #include "policy/fees.h" #include "streams.h" #include "timedata.h" #include "util.h" +#include "clientversion.h" #include "utilmoneystr.h" #include "utiltime.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, unsigned int _entryHeight, - bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): - tx(_tx), nFee(_nFee), nTime(_nTime), entryHeight(_entryHeight), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) + int64_t _nTime, double _entryPriority, unsigned int _entryHeight, + CAmount _inChainInputValue, bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp): + tx(_tx), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), + inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); + nModSize = tx->CalculateModifiedSize(GetTxSize()); nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); nCountWithDescendants = 1; @@ -60,6 +62,15 @@ size_t CTxMemPoolEntry::GetTxSize() const return GetVirtualTransactionSize(nTxWeight, sigOpCost); } +double CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const +{ + double deltaPriority = ((double)(currentHeight - entryHeight) * inChainInputValue) / nModSize; + double dResult = entryPriority + deltaPriority; + if (dResult < 0) // This should only happen if it wrapped + return 0.0; + return dResult; +} + // Update the given tx for any in-mempool descendants. // Assumes that setMemPoolChildren is correct for the given tx and all // descendants. @@ -332,8 +343,19 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } +CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : + nTransactionsUpdated(0), minerPolicyEstimator(new CBlockPolicyEstimator(_minReasonableRelayFee)), ownsEstimator(true) +{ + _clear(); //lock free clear + + // Sanity checks off by default for performance, because otherwise + // accepting transactions becomes O(N^2) where N is the number + // of transactions in the pool + nCheckFrequency = 0; +} + CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : - nTransactionsUpdated(0), minerPolicyEstimator(estimator) + nTransactionsUpdated(0), minerPolicyEstimator(estimator), ownsEstimator(false) { _clear(); //lock free clear @@ -343,6 +365,13 @@ CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : nCheckFrequency = 0; } +CTxMemPool::~CTxMemPool() +{ + if (ownsEstimator) { + delete minerPolicyEstimator; + } +} + bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); @@ -374,9 +403,9 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, // Update transaction for any feeDelta created by PrioritiseTransaction // TODO: refactor so that the fee delta is calculated before inserting // into mapTx. - std::map::const_iterator pos = mapDeltas.find(hash); + std::map >::const_iterator pos = mapDeltas.find(hash); if (pos != mapDeltas.end()) { - const CAmount &delta = pos->second; + const CAmount &delta = pos->second.second; if (delta) { mapTx.modify(newit, update_fee_delta(delta)); } @@ -442,7 +471,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -526,7 +555,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem continue; const Coin &coin = pcoins->AccessCoin(txin.prevout); if (nCheckFrequency != 0) assert(!coin.IsPruned()); - if (coin.IsPruned() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { + if (coin.IsPruned() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < Params().GetConsensus(coin.nHeight).nCoinbaseMaturity)) { txToRemove.insert(it); break; } @@ -619,7 +648,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (GetRand(std::numeric_limits::max()) >= nCheckFrequency) return; - LogPrint(BCLog::MEMPOOL, "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); + LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size()); uint64_t checkTotal = 0; uint64_t innerUsage = 0; @@ -706,7 +735,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const else { CValidationState state; bool fCheckResult = tx.IsCoinBase() || - Consensus::CheckTxInputs(tx, state, mempoolDuplicate, nSpendHeight); + Consensus::CheckTxInputs(Params(), tx, state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, 1000000); } @@ -722,7 +751,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { bool fCheckResult = entry->GetTx().IsCoinBase() || - Consensus::CheckTxInputs(entry->GetTx(), state, mempoolDuplicate, nSpendHeight); + Consensus::CheckTxInputs(Params(), entry->GetTx(), state, mempoolDuplicate, nSpendHeight); assert(fCheckResult); UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); stepsSinceLastRemove = 0; @@ -834,15 +863,74 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) +CFeeRate CTxMemPool::estimateFee(int nBlocks) const +{ + LOCK(cs); + return minerPolicyEstimator->estimateFee(nBlocks); +} + +CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const +{ + LOCK(cs); + return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); +} + +double CTxMemPool::estimatePriority(int nBlocks) const +{ + LOCK(cs); + return minerPolicyEstimator->estimatePriority(nBlocks); +} + +double CTxMemPool::estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks) const +{ + LOCK(cs); + return minerPolicyEstimator->estimateSmartPriority(nBlocks, answerFoundAtBlocks, *this); +} + +bool CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const +{ + try { + LOCK(cs); + fileout << FEEFILE_MIN_BACKCOMPAT_VERSION; + fileout << CLIENT_VERSION; + minerPolicyEstimator->Write(fileout); + } catch (const std::exception&) { + LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); + return false; + } + return true; +} + +bool CTxMemPool::ReadFeeEstimates(CAutoFile& filein) +{ + try { + int nVersionRequired, nVersionThatWrote; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); + if (nVersionThatWrote < FEEFILE_MIN_COMPAT_VERSION_WRITER) { + return error("CTxMemPool::ReadFeeEstimates(): cannot re-use fee estimate files from version %d, ignoring file (non-fatal)", nVersionThatWrote); + } + + LOCK(cs); + minerPolicyEstimator->Read(filein, nVersionThatWrote); + } catch (const std::exception&) { + LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); + return false; + } + return true; +} + +void CTxMemPool::PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta) { { LOCK(cs); - CAmount &delta = mapDeltas[hash]; - delta += nFeeDelta; + std::pair &deltas = mapDeltas[hash]; + deltas.first += dPriorityDelta; + deltas.second += nFeeDelta; txiter it = mapTx.find(hash); if (it != mapTx.end()) { - mapTx.modify(it, update_fee_delta(delta)); + mapTx.modify(it, update_fee_delta(deltas.second)); // Now update all ancestors' modified fees with descendants setEntries setAncestors; uint64_t nNoLimit = std::numeric_limits::max(); @@ -861,17 +949,18 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD ++nTransactionsUpdated; } } - LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); + LogPrintf("PrioritiseTransaction: %s priority += %f, fee += %d\n", strHash, dPriorityDelta, FormatMoney(nFeeDelta)); } -void CTxMemPool::ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const +void CTxMemPool::ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const { LOCK(cs); - std::map::const_iterator pos = mapDeltas.find(hash); + std::map >::const_iterator pos = mapDeltas.find(hash); if (pos == mapDeltas.end()) return; - const CAmount &delta = pos->second; - nFeeDelta += delta; + const std::pair &deltas = pos->second; + dPriorityDelta += deltas.first; + nFeeDelta += deltas.second; } void CTxMemPool::ClearPrioritisation(const uint256 hash) @@ -1059,7 +1148,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpends } if (maxFeeRateRemoved > CFeeRate(0)) { - LogPrint(BCLog::MEMPOOL, "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); + LogPrint("mempool", "Removed %u txn, rolling minimum fee bumped to %s\n", nTxnRemoved, maxFeeRateRemoved.ToString()); } } diff --git a/src/txmempool.h b/src/txmempool.h index c86859118a1..8c4978fd9ef 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,7 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" -#include "policy/feerate.h" +#include "policy/fees.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -30,6 +30,26 @@ class CAutoFile; class CBlockIndex; +/** Minimum client version supported in fee_estimates.dat + * Clients with a lower version than this should ignore the file + */ +static const int FEEFILE_MIN_BACKCOMPAT_VERSION = 1140700; // 1.14.7.0 + +/** Minimum client version required to read fee_estimates.dat + * Files created with a lower version than this are ignored + */ +static const int FEEFILE_MIN_COMPAT_VERSION_WRITER = 1140700; // 1.14.7.0 + +inline double AllowFreeThreshold() +{ + return COIN * 144 / 250; +} + +inline bool AllowFree(double dPriority) +{ + return dPriority > AllowFreeThreshold(); +} + /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -68,9 +88,12 @@ class CTxMemPoolEntry CTransactionRef tx; CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) + size_t nModSize; //!< ... and modified size for priority size_t nUsageSize; //!< ... and total memory usage int64_t nTime; //!< Local time when entering the mempool + double entryPriority; //!< Priority when entering the mempool unsigned int entryHeight; //!< Chain height when entering the mempool + CAmount inChainInputValue; //!< Sum of all txin values that are already in blockchain bool spendsCoinbase; //!< keep track of transactions that spend a coinbase int64_t sigOpCost; //!< Total sigop cost int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block @@ -91,14 +114,19 @@ class CTxMemPoolEntry public: CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, unsigned int _entryHeight, - bool spendsCoinbase, + int64_t _nTime, double _entryPriority, unsigned int _entryHeight, + CAmount _inChainInputValue, bool spendsCoinbase, int64_t nSigOpsCost, LockPoints lp); CTxMemPoolEntry(const CTxMemPoolEntry& other); const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } + /** + * Fast calculation of lower bound of current priority as update + * from entry priority. Only inputs that were originally in-chain will age. + */ + double GetPriority(unsigned int currentHeight) const; const CAmount& GetFee() const { return nFee; } size_t GetTxSize() const; size_t GetTxWeight() const { return nTxWeight; } @@ -202,7 +230,7 @@ struct mempoolentry_txid class CompareTxMemPoolEntryByDescendantScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); @@ -224,7 +252,7 @@ class CompareTxMemPoolEntryByDescendantScore } // Calculate which score to use for an entry (avoiding division). - bool UseDescendantScore(const CTxMemPoolEntry &a) + bool UseDescendantScore(const CTxMemPoolEntry &a) const { double f1 = (double)a.GetModifiedFee() * a.GetSizeWithDescendants(); double f2 = (double)a.GetModFeesWithDescendants() * a.GetTxSize(); @@ -239,7 +267,7 @@ class CompareTxMemPoolEntryByDescendantScore class CompareTxMemPoolEntryByScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { double f1 = (double)a.GetModifiedFee() * b.GetTxSize(); double f2 = (double)b.GetModifiedFee() * a.GetTxSize(); @@ -253,7 +281,7 @@ class CompareTxMemPoolEntryByScore class CompareTxMemPoolEntryByEntryTime { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { return a.GetTime() < b.GetTime(); } @@ -262,7 +290,7 @@ class CompareTxMemPoolEntryByEntryTime class CompareTxMemPoolEntryByAncestorFee { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { double aFees = a.GetModFeesWithAncestors(); double aSize = a.GetSizeWithAncestors(); @@ -413,6 +441,7 @@ class CTxMemPool uint32_t nCheckFrequency; //!< Value n means that n times in 2^32 we check. unsigned int nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; + bool ownsEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. uint64_t cachedInnerUsage; //!< sum of dynamic memory usage of all the map elements (NOT the maps themselves) @@ -492,11 +521,13 @@ class CTxMemPool public: indirectmap mapNextTx; - std::map mapDeltas; + std::map > mapDeltas; /** Create a new CTxMemPool. */ + CTxMemPool(const CFeeRate& _minReasonableRelayFee); CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); + ~CTxMemPool(); /** * If sanity-checking is turned on, check makes sure the pool is @@ -533,8 +564,8 @@ class CTxMemPool bool HasNoInputsOf(const CTransaction& tx) const; /** Affect CreateNewBlock prioritisation of transactions */ - void PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta); - void ApplyDelta(const uint256 hash, CAmount &nFeeDelta) const; + void PrioritiseTransaction(const uint256 hash, const std::string strHash, double dPriorityDelta, const CAmount& nFeeDelta); + void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta) const; void ClearPrioritisation(const uint256 hash); public: @@ -624,6 +655,28 @@ class CTxMemPool TxMempoolInfo info(const uint256& hash) const; std::vector infoAll() const; + /** Estimate fee rate needed to get into the next nBlocks + * If no answer can be given at nBlocks, return an estimate + * at the lowest number of blocks where one can be given + */ + CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; + + /** Estimate fee rate needed to get into the next nBlocks */ + CFeeRate estimateFee(int nBlocks) const; + + /** Estimate priority needed to get into the next nBlocks + * If no answer can be given at nBlocks, return an estimate + * at the lowest number of blocks where one can be given + */ + double estimateSmartPriority(int nBlocks, int *answerFoundAtBlocks = NULL) const; + + /** Estimate priority needed to get into the next nBlocks */ + double estimatePriority(int nBlocks) const; + + /** Write/Read estimates to disk */ + bool WriteFeeEstimates(CAutoFile& fileout) const; + bool ReadFeeEstimates(CAutoFile& filein); + size_t DynamicMemoryUsage() const; boost::signals2::signal NotifyEntryAdded; @@ -683,4 +736,17 @@ class CCoinsViewMemPool : public CCoinsViewBacked bool HaveCoins(const COutPoint &outpoint) const; }; +// We want to sort transactions by coin age priority +typedef std::pair TxCoinAgePriority; + +struct TxCoinAgePriorityCompare +{ + bool operator()(const TxCoinAgePriority& a, const TxCoinAgePriority& b) + { + if (a.first == b.first) + return CompareTxMemPoolEntryByScore()(*(b.second), *(a.second)); // Reverse order to make sort less than + return a.first < b.first; + } +}; + #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/validation.cpp b/src/validation.cpp index 7aaed05f63e..e943e1a1ff0 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include "init.h" #include "policy/fees.h" #include "policy/policy.h" +#include "policy/fees.h" #include "pow.h" #if ENABLE_LIBOQS #include "pqc/pqc_commitment.h" @@ -91,7 +92,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFeeRate = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool(::minRelayTxFeeRate); +CBlockPolicyEstimator mempoolEstimator(::minRelayTxFeeRate); +CTxMemPool mempool(&mempoolEstimator); /** * Returns true if there are nRequired or more blocks of minVersion or above @@ -762,8 +764,16 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C double nPriorityDummy = 0; pool.ApplyDeltas(hash, nPriorityDummy, nModifiedFees); - CAmount inChainInputValue; - double dPriority = view.GetPriority(tx, chainActive.Height(), inChainInputValue); + CAmount inChainInputValue = 0; + double dPriorityInputs = 0.0; + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (!coin.IsPruned() && coin.nHeight <= chainActive.Height()) { + dPriorityInputs += (double)coin.out.nValue * (chainActive.Height() - coin.nHeight); + inChainInputValue += coin.out.nValue; + } + } + double dPriority = tx.ComputePriority(dPriorityInputs); // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. @@ -1439,22 +1449,22 @@ bool CheckTxInputs(const CChainParams& params, const CTransaction& tx, CValidati for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsPruned()); // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { + if (coin.IsCoinBase()) { // Dogecoin: Switch maturity at depth 145,000 - int nCoinbaseMaturity = params.GetConsensus(coins->nHeight).nCoinbaseMaturity; - if (nSpendHeight - coins->nHeight < nCoinbaseMaturity) + int nCoinbaseMaturity = params.GetConsensus(coin.nHeight).nCoinbaseMaturity; + if (nSpendHeight - coin.nHeight < nCoinbaseMaturity) return state.Invalid(false, REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); } // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } From b2ed44a7302692df221d60f4b78c3998c63bd61a Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Sat, 13 Jun 2026 03:04:08 +0000 Subject: [PATCH 23/32] Session 3: fix coins_tests.cpp for per-txout and 21180ff73 semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename GetCoins→GetCoin, remove HaveCoins override in CCoinsViewTest so it properly overrides the virtual base method - Add HaveCoin testing to coins_cache_simulation_test (per 21180ff73) - Update CheckAccessCoin(PRUNED,ABSENT) to expect ABSENT/NO_ENTRY now that GetCoin returns false for spent coins in child caches --- src/bitcoin-tx.cpp | 4 +- src/coins.cpp | 129 +++++++---------- src/coins.h | 277 +++++-------------------------------- src/init.cpp | 11 +- src/net_processing.cpp | 4 +- src/policy/policy.cpp | 4 +- src/qt/transactiondesc.cpp | 2 +- src/rest.cpp | 2 +- src/rpc/blockchain.cpp | 9 +- src/rpc/rawtransaction.cpp | 8 +- src/test/coins_tests.cpp | 62 +++++---- src/txdb.cpp | 122 ++++++++++++++-- src/txdb.h | 10 +- src/txmempool.cpp | 14 +- src/txmempool.h | 4 +- src/validation.cpp | 55 ++++---- src/validation.h | 4 + 17 files changed, 297 insertions(+), 424 deletions(-) diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 15f788bf15e..b62b1621728 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -582,7 +582,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) { const Coin& coin = view.AccessCoin(out); - if (!coin.IsPruned() && coin.out.scriptPubKey != scriptPubKey) { + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); @@ -618,7 +618,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; const Coin& coin = view.AccessCoin(txin.prevout); - if (coin.IsPruned()) { + if (coin.IsSpent()) { fComplete = false; continue; } diff --git a/src/coins.cpp b/src/coins.cpp index 90949087508..27a24354785 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -10,48 +10,21 @@ #include -/** - * calculate number of bytes for the bitmask, and its number of non-zero bytes - * each bit in the bitmask represents the availability of one output, but the - * availabilities of the first two outputs are encoded separately - */ -void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const { - unsigned int nLastUsedByte = 0; - for (unsigned int b = 0; 2+b*8 < vout.size(); b++) { - bool fZero = true; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) { - if (!vout[2+b*8+i].IsNull()) { - fZero = false; - continue; - } - } - if (!fZero) { - nLastUsedByte = b + 1; - nNonzeroBytes++; - } - } - nBytes += nLastUsedByte; -} - -bool CCoins::Spend(uint32_t nPos) -{ - if (nPos >= vout.size() || vout[nPos].IsNull()) - return false; - vout[nPos].SetNull(); - Cleanup(); - return true; -} - -bool CCoinsView::GetCoins(const COutPoint &outpoint, Coin &coin) const { return false; } -bool CCoinsView::HaveCoins(const COutPoint &outpoint) const { return false; } +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const +{ + Coin coin; + return GetCoin(outpoint, coin); +} + CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const COutPoint &outpoint, Coin &coin) const { return base->GetCoins(outpoint, coin); } -bool CCoinsViewBacked::HaveCoins(const COutPoint &outpoint) const { return base->HaveCoins(outpoint); } +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } @@ -66,51 +39,51 @@ size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::iterator CCoinsViewCache::FetchCoins(const COutPoint &outpoint) const { +CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; Coin tmp; - if (!base->GetCoins(outpoint, tmp)) + if (!base->GetCoin(outpoint, tmp)) return cacheCoins.end(); CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; - if (ret->second.coins.IsPruned()) { + if (ret->second.coin.IsSpent()) { // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const COutPoint &outpoint, Coin &coin) const { - CCoinsMap::const_iterator it = FetchCoins(outpoint); +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it != cacheCoins.end()) { - coin = it->second.coins; - return true; + coin = it->second.coin; + return !coin.IsSpent(); } return false; } void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { - assert(!coin.IsPruned()); + assert(!coin.IsSpent()); if (coin.out.scriptPubKey.IsUnspendable()) return; CCoinsMap::iterator it; bool inserted; std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); bool fresh = false; if (!inserted) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); } if (!possible_overwrite) { - if (!it->second.coins.IsPruned()) { + if (!it->second.coin.IsSpent()) { throw std::logic_error("Adding new coin that replaces non-pruned entry"); } fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } - it->second.coins = std::move(coin); + it->second.coin = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); - cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { @@ -123,38 +96,39 @@ void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { } } -void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { - CCoinsMap::iterator it = FetchCoins(outpoint); - if (it == cacheCoins.end()) return; - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); +bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) return false; + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); if (moveout) { - *moveout = std::move(it->second.coins); + *moveout = std::move(it->second.coin); } if (it->second.flags & CCoinsCacheEntry::FRESH) { cacheCoins.erase(it); } else { it->second.flags |= CCoinsCacheEntry::DIRTY; - it->second.coins.Clear(); + it->second.coin.Clear(); } + return true; } static const Coin coinEmpty; const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { - CCoinsMap::const_iterator it = FetchCoins(outpoint); + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { return coinEmpty; } else { - return it->second.coins; + return it->second.coin; } } -bool CCoinsViewCache::HaveCoins(const COutPoint &outpoint) const { - CCoinsMap::const_iterator it = FetchCoins(outpoint); - return (it != cacheCoins.end() && !it->second.coins.IsPruned()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } -bool CCoinsViewCache::HaveCoinsInCache(const COutPoint &outpoint) const { +bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -176,12 +150,12 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coins.IsPruned())) { + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins = std::move(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache @@ -194,21 +168,21 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coins.IsPruned()) + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins = std::move(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -236,7 +210,7 @@ void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -245,13 +219,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const -{ - const Coin& coin = AccessCoin(input.prevout); - assert(!coin.IsPruned()); - return coin.out; -} - CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const { if (tx.IsCoinBase()) @@ -259,7 +226,7 @@ CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const CAmount nResult = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += GetOutputFor(tx.vin[i]).nValue; + nResult += AccessCoin(tx.vin[i].prevout).out.nValue; return nResult; } @@ -268,7 +235,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - if (!HaveCoins(tx.vin[i].prevout)) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -283,7 +250,7 @@ const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) COutPoint iter(txid, 0); while (iter.n < MAX_OUTPUTS_PER_BLOCK) { const Coin& alternate = view.AccessCoin(iter); - if (!alternate.IsPruned()) return alternate; + if (!alternate.IsSpent()) return alternate; ++iter.n; } return coinEmpty; diff --git a/src/coins.h b/src/coins.h index 8fd63c3fb2f..0f70019b1b3 100644 --- a/src/coins.h +++ b/src/coins.h @@ -15,6 +15,7 @@ #include "uint256.h" #include +#include #include #include @@ -30,18 +31,18 @@ class Coin { public: - //! whether the containing transaction was a coinbase - bool fCoinBase; - //! unspent transaction output CTxOut out; - //! at which height the containing transaction was included in the active block chain - uint32_t nHeight; + //! whether containing transaction was a coinbase + unsigned int fCoinBase : 1; + + //! at which height this containing transaction was included in the active block chain + uint32_t nHeight : 31; - //! construct a Coin from a CTxOut and height/coinbase properties. - Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : fCoinBase(fCoinBaseIn), out(std::move(outIn)), nHeight(nHeightIn) {} - Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : fCoinBase(fCoinBaseIn), out(outIn), nHeight(nHeightIn) {} + //! construct a Coin from a CTxOut and height/coinbase information. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} void Clear() { out.SetNull(); @@ -58,7 +59,7 @@ class Coin template void Serialize(Stream &s) const { - assert(!IsPruned()); + assert(!IsSpent()); uint32_t code = nHeight * 2 + fCoinBase; ::Serialize(s, VARINT(code)); ::Serialize(s, CTxOutCompressor(REF(out))); @@ -73,7 +74,7 @@ class Coin ::Unserialize(s, REF(CTxOutCompressor(out))); } - bool IsPruned() const { + bool IsSpent() const { return out.IsNull(); } @@ -82,223 +83,6 @@ class Coin } }; -/** - * Pruned version of CTransaction: only retains metadata and unspent transaction outputs - * - * Serialized format: - * - VARINT(nVersion) - * - VARINT(nCode) - * - unspentness bitvector, for vout[2] and further; least significant byte first - * - the non-spent CTxOuts (via CTxOutCompressor) - * - VARINT(nHeight) - * - * The nCode value consists of: - * - bit 0: IsCoinBase() - * - bit 1: vout[0] is not spent - * - bit 2: vout[1] is not spent - * - The higher bits encode N, the number of non-zero bytes in the following bitvector. - * - In case both bit 1 and bit 2 are unset, they encode N-1, as there must be at - * least one non-spent output). - * - * Example: 0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e - * <><><--------------------------------------------><----> - * | \ | / - * version code vout[1] height - * - * - version = 1 - * - code = 4 (vout[1] is not spent, and 0 non-zero bytes of bitvector follow) - * - unspentness bitvector: as 0 non-zero bytes follow, it has length 0 - * - vout[1]: 835800816115944e077fe7c803cfa57f29b36bf87c1d35 - * * 8358: compact amount representation for 60000000000 (600 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 816115944e077fe7c803cfa57f29b36bf87c1d35: address uint160 - * - height = 203998 - * - * - * Example: 0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b - * <><><--><--------------------------------------------------><----------------------------------------------><----> - * / \ \ | | / - * version code unspentness vout[4] vout[16] height - * - * - version = 1 - * - code = 9 (coinbase, neither vout[0] or vout[1] are unspent, - * 2 (1, +1 because both bit 1 and bit 2 are unset) non-zero bitvector bytes follow) - * - unspentness bitvector: bits 2 (0x04) and 14 (0x4000) are set, so vout[2+2] and vout[14+2] are unspent - * - vout[4]: 86ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4ee - * * 86ef97d579: compact amount representation for 234925952 (2.35 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 61b01caab50f1b8e9c50a5057eb43c2d9563a4ee: address uint160 - * - vout[16]: bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4 - * * bbd123: compact amount representation for 110397 (0.001 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 8c988f1a4a4de2161e0f50aac7f17e7f9555caa4: address uint160 - * - height = 120891 - */ -class CCoins -{ -public: - //! whether transaction is a coinbase - bool fCoinBase; - - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector vout; - - //! at which height this transaction was included in the active block chain - int nHeight; - - void FromTx(const CTransaction &tx, int nHeightIn) { - fCoinBase = tx.IsCoinBase(); - vout = tx.vout; - nHeight = nHeightIn; - ClearUnspendable(); - } - - //! construct a CCoins from a CTransaction, at a given height - CCoins(const CTransaction &tx, int nHeightIn) { - FromTx(tx, nHeightIn); - } - - void Clear() { - fCoinBase = false; - std::vector().swap(vout); - nHeight = 0; - } - - //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0) { } - - //!remove spent outputs at the end of vout - void Cleanup() { - while (vout.size() > 0 && vout.back().IsNull()) - vout.pop_back(); - if (vout.empty()) - std::vector().swap(vout); - } - - void ClearUnspendable() { - BOOST_FOREACH(CTxOut &txout, vout) { - if (txout.scriptPubKey.IsUnspendable()) - txout.SetNull(); - } - Cleanup(); - } - - void swap(CCoins &to) { - std::swap(to.fCoinBase, fCoinBase); - to.vout.swap(vout); - std::swap(to.nHeight, nHeight); - } - - //! equality test - friend bool operator==(const CCoins &a, const CCoins &b) { - // Empty CCoins objects are always equal. - if (a.IsPruned() && b.IsPruned()) - return true; - return a.fCoinBase == b.fCoinBase && - a.nHeight == b.nHeight && - a.vout == b.vout; - } - friend bool operator!=(const CCoins &a, const CCoins &b) { - return !(a == b); - } - - void CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const; - - bool IsCoinBase() const { - return fCoinBase; - } - - template - void Serialize(Stream &s) const { - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - int nVersionDummy = 0; - ::Serialize(s, VARINT(nVersionDummy)); - // header code - ::Serialize(s, VARINT(nCode)); - // spentness bitmask - for (unsigned int b = 0; b - void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - int nVersionDummy; - ::Unserialize(s, VARINT(nVersionDummy)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); - } - // coinbase height - ::Unserialize(s, VARINT(nHeight)); - Cleanup(); - } - - //! mark a vout spent - bool Spend(uint32_t nPos); - - //! check whether a particular output is still available - bool IsAvailable(unsigned int nPos) const { - return (nPos < vout.size() && !vout[nPos].IsNull()); - } - - //! check whether the entire CCoins is spent - //! note that only !IsPruned() CCoins can be serialized - bool IsPruned() const { - BOOST_FOREACH(const CTxOut &out, vout) - if (!out.IsNull()) - return false; - return true; - } - - size_t DynamicMemoryUsage() const { - size_t ret = memusage::DynamicUsage(vout); - BOOST_FOREACH(const CTxOut &out, vout) { - ret += RecursiveDynamicUsage(out.scriptPubKey); - } - return ret; - } -}; - class SaltedOutpointHasher { private: @@ -320,7 +104,7 @@ class SaltedOutpointHasher struct CCoinsCacheEntry { - Coin coins; // The actual cached data. + Coin coin; // The actual cached data. unsigned char flags; enum Flags { @@ -334,7 +118,7 @@ struct CCoinsCacheEntry }; CCoinsCacheEntry() : flags(0) {} - explicit CCoinsCacheEntry(Coin&& coin_) : coins(std::move(coin_)), flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} }; typedef std::unordered_map CCoinsMap; @@ -363,12 +147,14 @@ class CCoinsViewCursor class CCoinsView { public: - //! Retrieve the Coin (unspent transaction output) for a given outpoint. - virtual bool GetCoins(const COutPoint &outpoint, Coin &coin) const; + /** Retrieve the Coin (unspent transaction output) for a given outpoint. + * Returns true only when an unspent coin was found, which is returned in coin. + * When false is returned, coin's value is unspecified. + */ + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given outpoint. - //! This may (but cannot always) return true for spent outputs. - virtual bool HaveCoins(const COutPoint &outpoint) const; + //! Just check whether a given outpoint is unspent. + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; @@ -396,8 +182,8 @@ class CCoinsViewBacked : public CCoinsView public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const COutPoint &outpoint, Coin &coin) const override; - bool HaveCoins(const COutPoint &outpoint) const override; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; @@ -424,22 +210,25 @@ class CCoinsViewCache : public CCoinsViewBacked CCoinsViewCache(CCoinsView *baseIn); // Standard CCoinsView methods - bool GetCoins(const COutPoint &outpoint, Coin &coin) const; - bool HaveCoins(const COutPoint &outpoint) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); + CCoinsViewCursor* Cursor() const { + throw std::logic_error("CCoinsViewCache cursor iteration not supported."); + } /** * Check if we have the given utxo already loaded in this cache. - * The semantics are the same as HaveCoins(), but no calls to + * The semantics are the same as HaveCoin(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const COutPoint &outpoint) const; + bool HaveCoinInCache(const COutPoint &outpoint) const; /** * Return a reference to Coin in the cache, or a pruned one if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are + * more efficient than GetCoin. Modifications to other cache entries are * allowed while accessing the returned pointer. */ const Coin& AccessCoin(const COutPoint &output) const; @@ -455,7 +244,7 @@ class CCoinsViewCache : public CCoinsViewBacked * If no unspent output exists for the passed outpoint, this call * has no effect. */ - void SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); + bool SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** * Push the modifications applied to this cache to its base. @@ -489,10 +278,8 @@ class CCoinsViewCache : public CCoinsViewBacked //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - const CTxOut &GetOutputFor(const CTxIn& input) const; - private: - CCoinsMap::iterator FetchCoins(const COutPoint &outpoint) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. diff --git a/src/init.cpp b/src/init.cpp index 1e43b36f7bb..790348a19b2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -152,9 +152,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const COutPoint &outpoint, Coin &coin) const override { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(outpoint, coin); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); @@ -168,7 +168,6 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked // Writes do not need similar protection, as failure to write is handled by the caller. }; -static CCoinsViewDB *pcoinsdbview = NULL; static CCoinsViewErrorCatcher *pcoinscatcher = NULL; static std::unique_ptr globalVerifyHandle; @@ -1503,6 +1502,12 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); + } else { + // If necessary, upgrade from older database format. + if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; + } } if (!LoadBlockIndex(chainparams)) { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index bd514dce8e2..eaad8290b73 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -990,8 +990,8 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 - pcoinsTip->HaveCoinsInCache(COutPoint(inv.hash, 1)); + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index f90d6901697..5426a151443 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -128,7 +128,7 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector > vSolutions; txnouttype whichType; @@ -167,7 +167,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) if (tx.vin[i].scriptWitness.IsNull()) continue; - const CTxOut &prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut &prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; // get the scriptPubKey corresponding to this input: CScript prevScript = prev.scriptPubKey; diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 84cdc65b6fb..97392ad7615 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -791,7 +791,7 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco COutPoint prevout = txin.prevout; Coin prev; - if(pcoinsTip->GetCoins(prevout, prev)) + if(pcoinsTip->GetCoin(prevout, prev)) { { strHTML += "
  • "; diff --git a/src/rest.cpp b/src/rest.cpp index 1abd8fbb762..ff326d63eb8 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -521,7 +521,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) for (size_t i = 0; i < vOutPoints.size(); i++) { bool hit = false; Coin coin; - if (view.GetCoins(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { hit = true; outs.emplace_back(std::move(coin)); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a8a9310354d..26c2bb288d6 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -20,6 +20,7 @@ #include "rpc/server.h" #include "streams.h" #include "sync.h" +#include "txdb.h" #include "txmempool.h" #include "undo.h" #include "util.h" @@ -999,7 +1000,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsTip, stats)) { + if (GetUTXOStats(pcoinsdbview, stats)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); ret.pushKV("transactions", (int64_t)stats.nTransactions); @@ -1066,13 +1067,13 @@ UniValue gettxout(const JSONRPCRequest& request) if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(out, coin) || mempool.isSpent(out)) // TODO: this should be done by the CCoinsViewMemPool + if (!view.GetCoin(out, coin) || mempool.isSpent(out)) // TODO: this should be done by the CCoinsViewMemPool return NullUniValue; } else { - if (!pcoinsTip->GetCoins(out, coin)) + if (!pcoinsTip->GetCoin(out, coin)) return NullUniValue; } - if (coin.IsPruned()) + if (coin.IsSpent()) return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 8ac5260160c..46531ad7315 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -328,7 +328,7 @@ UniValue gettxoutproof(const JSONRPCRequest& request) pblockindex = mapBlockIndex[hashBlock]; } else { const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); - if (!coin.IsPruned() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + if (!coin.IsSpent() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { pblockindex = chainActive[coin.nHeight]; } } @@ -794,7 +794,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) { const Coin& coin = view.AccessCoin(out); - if (!coin.IsPruned() && coin.out.scriptPubKey != scriptPubKey) { + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); @@ -866,7 +866,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; const Coin& coin = view.AccessCoin(txin.prevout); - if (coin.IsPruned()) { + if (coin.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } @@ -946,7 +946,7 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) bool fHaveChain = false; for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); - fHaveChain = !existingCoin.IsPruned(); + fHaveChain = !existingCoin.IsSpent(); } bool fHaveMempool = mempool.exists(hashTx); if (!fHaveMempool && !fHaveChain) { diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 9fdf60cd787..a4eaa1595a0 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -27,7 +27,7 @@ namespace //! equality test bool operator==(const Coin &a, const Coin &b) { // Empty Coin objects are always equal. - if (a.IsPruned() && b.IsPruned()) return true; + if (a.IsSpent() && b.IsSpent()) return true; return a.fCoinBase == b.fCoinBase && a.nHeight == b.nHeight && a.out == b.out; @@ -39,35 +39,29 @@ class CCoinsViewTest : public CCoinsView std::map map_; public: - bool GetCoins(const COutPoint& outpoint, Coin& coin) const + bool GetCoin(const COutPoint& outpoint, Coin& coin) const override { std::map::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } coin = it->second; - if (coin.IsPruned() && insecure_rand() % 2 == 0) { + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const COutPoint& outpoint) const - { - Coin coin; - return GetCoins(outpoint, coin); - } - - uint256 GetBestBlock() const { return hashBestBlock_; } + uint256 GetBestBlock() const override { return hashBestBlock_; } bool BatchWrite(CCoinsMap& mapCoins, const uint256& hashBlock) { for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -90,7 +84,7 @@ class CCoinsViewCacheTest : public CCoinsViewCache // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); } BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -145,11 +139,25 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) { uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. Coin& coin = result[COutPoint(txid, 0)]; + + // Determine whether to test HaveCoin before or after Access* (or both). As these functions + // can influence each other's behaviour by pulling things into the cache, all combinations + // are tested. + bool test_havecoin_before = insecure_rand() % 4 == 0; + bool test_havecoin_after = insecure_rand() % 4 == 0; + + bool result_havecoin = test_havecoin_before ? stack.back()->HaveCoin(COutPoint(txid, 0)) : false; const Coin& entry = stack.back()->AccessCoin(COutPoint(txid, 0)); BOOST_CHECK(coin == entry); + BOOST_CHECK(!test_havecoin_before || result_havecoin == !entry.IsSpent()); + + if (test_havecoin_after) { + bool ret = stack.back()->HaveCoin(COutPoint(txid, 0)); + BOOST_CHECK(ret == !entry.IsSpent()); + } - if (insecure_rand() % 5 == 0 || coin.IsPruned()) { - if (coin.IsPruned()) { + if (insecure_rand() % 5 == 0 || coin.IsSpent()) { + if (coin.IsSpent()) { added_an_entry = true; } else { updated_an_entry = true; @@ -160,7 +168,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) coin.Clear(); removed_an_entry = true; } - if (coin.IsPruned()) { + if (coin.IsSpent()) { stack.back()->SpendCoin(COutPoint(txid, 0)); } else { stack.back()->AddCoin(COutPoint(txid, 0), Coin(coin), true); @@ -172,7 +180,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) for (auto it = result.begin(); it != result.end(); it++) { const Coin& coin = stack.back()->AccessCoin(it->first); BOOST_CHECK(coin == it->second); - if (coin.IsPruned()) { + if (coin.IsSpent()) { missed_an_entry = true; } else { found_an_entry = true; @@ -278,7 +286,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate unsigned int height = insecure_rand(); - Coin oldcoins; + Coin old_coin; // 2/20 times create a new coinbase if (randiter % 20 < 2 || coinbaseids.size() < 10) { @@ -330,7 +338,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevout]; + old_coin = result[prevout]; // Update the expected result of prevouthash to know these coins are spent result[prevout].Clear(); @@ -356,7 +364,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) utxoset.insert(outpoint); // Track this tx and undo info to use later - utxoData.emplace(outpoint, std::make_tuple(tx,undo,oldcoins)); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,old_coin)); } else if (utxoset.size()) { //1/20 times undo a previous transaction auto utxod = FindRandomFrom(utxoset); @@ -505,11 +513,11 @@ void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); coin.Clear(); - assert(coin.IsPruned()); + assert(coin.IsSpent()); if (value != PRUNED) { coin.out.nValue = value; coin.nHeight = 1; - assert(!coin.IsPruned()); + assert(!coin.IsSpent()); } } @@ -522,10 +530,10 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); + SetCoinsValue(value, entry.coin); auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) @@ -535,10 +543,10 @@ void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - value = it->second.coins.out.nValue; + value = it->second.coin.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); @@ -597,7 +605,7 @@ BOOST_AUTO_TEST_CASE(ccoins_access) CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); + CheckAccessCoin(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); diff --git a/src/txdb.cpp b/src/txdb.cpp index 02f9dbd86ac..88494b762c5 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -27,10 +27,10 @@ static const char DB_LAST_BLOCK = 'l'; namespace { -struct CoinsEntry { +struct CoinEntry { COutPoint* outpoint; char key; - CoinsEntry(const COutPoint* ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} + CoinEntry(const COutPoint* ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} template void Serialize(Stream &s) const { @@ -53,12 +53,12 @@ CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(Get { } -bool CCoinsViewDB::GetCoins(const COutPoint &outpoint, Coin &coin) const { - return db.Read(CoinsEntry(&outpoint), coin); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const COutPoint &outpoint) const { - return db.Exists(CoinsEntry(&outpoint)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -74,11 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - CoinsEntry entry(&it->first); - if (it->second.coins.IsPruned()) + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) batch.Erase(entry); else - batch.Write(entry, it->second.coins); + batch.Write(entry, it->second.coin); changed++; } count++; @@ -128,7 +128,7 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const i->pcursor->Seek(DB_COIN); // Cache key of first record if (i->pcursor->Valid()) { - CoinsEntry entry(&i->keyTmp.second); + CoinEntry entry(&i->keyTmp.second); i->pcursor->GetKey(entry); i->keyTmp.first = entry.key; } else { @@ -165,7 +165,7 @@ bool CCoinsViewDBCursor::Valid() const void CCoinsViewDBCursor::Next() { pcursor->Next(); - CoinsEntry entry(&keyTmp.second); + CoinEntry entry(&keyTmp.second); if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false } else { @@ -250,3 +250,103 @@ bool CBlockTreeDB::LoadBlockIndexGuts(std::function vout; + + //! at which height this transaction was included in the active block chain + int nHeight; + + //! empty constructor + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } + + template + void Unserialize(Stream &s) { + unsigned int nCode = 0; + // version + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + // header code + ::Unserialize(s, VARINT(nCode)); + fCoinBase = nCode & 1; + std::vector vAvail(2, false); + vAvail[0] = (nCode & 2) != 0; + vAvail[1] = (nCode & 4) != 0; + unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); + // spentness bitmask + while (nMaskCode > 0) { + unsigned char chAvail = 0; + ::Unserialize(s, chAvail); + for (unsigned int p = 0; p < 8; p++) { + bool f = (chAvail & (1 << p)) != 0; + vAvail.push_back(f); + } + if (chAvail != 0) + nMaskCode--; + } + // txouts themself + vout.assign(vAvail.size(), CTxOut()); + for (unsigned int i = 0; i < vAvail.size(); i++) { + if (vAvail[i]) + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); + } + // coinbase height + ::Unserialize(s, VARINT(nHeight)); + } +}; + +} + +/** Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_COINS) { + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + pcursor->Next(); + } else { + break; + } + } + db.WriteBatch(batch); + return true; +} diff --git a/src/txdb.h b/src/txdb.h index 12c08664896..85618e4db7a 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,9 +23,7 @@ class uint256; //! Compensate for extra memory peak (x1.5-x1.9) at flush time. static constexpr int DB_PEAK_USAGE_FACTOR = 2; //! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; -//! Always periodic flush if less than this much space still available. -static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; +static constexpr int MAX_BLOCK_COINSDB_USAGE = 10 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) @@ -74,12 +72,14 @@ class CCoinsViewDB : public CCoinsView public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const COutPoint &outpoint, Coin &coin) const override; - bool HaveCoins(const COutPoint &outpoint) const override; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; uint256 GetBestBlock() const override; bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; CCoinsViewCursor *Cursor() const override; + //! Attempt to update from an older database format. Returns whether an error occurred. + bool Upgrade(); size_t EstimateSize() const override; }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 7797783aec5..cfcd4e20828 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -554,8 +554,8 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem if (it2 != mapTx.end()) continue; const Coin &coin = pcoins->AccessCoin(txin.prevout); - if (nCheckFrequency != 0) assert(!coin.IsPruned()); - if (coin.IsPruned() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < Params().GetConsensus(coin.nHeight).nCoinbaseMaturity)) { + if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < Params().GetConsensus(coin.nHeight).nCoinbaseMaturity)) { txToRemove.insert(it); break; } @@ -683,7 +683,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - assert(pcoins->HaveCoins(txin.prevout)); + assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -979,7 +979,7 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const COutPoint &outpoint, Coin &coin) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. @@ -992,11 +992,11 @@ bool CCoinsViewMemPool::GetCoins(const COutPoint &outpoint, Coin &coin) const { return false; } } - return (base->GetCoins(outpoint, coin) && !coin.IsPruned()); + return (base->GetCoin(outpoint, coin) && !coin.IsSpent()); } -bool CCoinsViewMemPool::HaveCoins(const COutPoint &outpoint) const { - return mempool.exists(outpoint) || base->HaveCoins(outpoint); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { diff --git a/src/txmempool.h b/src/txmempool.h index 8c4978fd9ef..f1c38d85638 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -732,8 +732,8 @@ class CCoinsViewMemPool : public CCoinsViewBacked public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const COutPoint &outpoint, Coin &coin) const; - bool HaveCoins(const COutPoint &outpoint) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; }; // We want to sort transactions by coin age priority diff --git a/src/validation.cpp b/src/validation.cpp index e943e1a1ff0..15603ab4551 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -247,6 +247,7 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc return chain.Genesis(); } +CCoinsViewDB *pcoinsdbview = NULL; CCoinsViewCache *pcoinsTip = NULL; CBlockTreeDB *pblocktree = NULL; @@ -437,7 +438,7 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; Coin coin; - if (!viewMemPool.GetCoins(txin.prevout, coin)) { + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } if (coin.nHeight == MEMPOOL_HEIGHT) { @@ -500,7 +501,7 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in unsigned int nSigOps = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; if (prevout.scriptPubKey.IsPayToScriptHash()) nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); } @@ -520,7 +521,7 @@ int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& i for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); } return nSigOps; @@ -613,7 +614,7 @@ static bool IsCurrentForFeeEstimation() bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& coins_to_uncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -708,10 +709,10 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // do we already have it? for (size_t out = 0; out < tx.vout.size(); out++) { COutPoint outpoint(hash, out); - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(outpoint); - if (view.HaveCoins(outpoint)) { - if (!fHadTxInCache) { - vHashTxnToUncache.push_back(outpoint); + bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); + if (view.HaveCoin(outpoint)) { + if (!had_coin_in_cache) { + coins_to_uncache.push_back(outpoint); } return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); } @@ -719,10 +720,10 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // do all inputs exist? BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout)) { - vHashTxnToUncache.push_back(txin.prevout); + if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); } - if (!view.HaveCoins(txin.prevout)) { + if (!view.HaveCoin(txin.prevout)) { if (pfMissingInputs) { *pfMissingInputs = true; } @@ -768,7 +769,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C double dPriorityInputs = 0.0; BOOST_FOREACH(const CTxIn& txin, tx.vin) { const Coin& coin = view.AccessCoin(txin.prevout); - if (!coin.IsPruned() && coin.nHeight <= chainActive.Height()) { + if (!coin.IsSpent() && coin.nHeight <= chainActive.Height()) { dPriorityInputs += (double)coin.out.nValue * (chainActive.Height() - coin.nHeight); inChainInputValue += coin.out.nValue; } @@ -1095,10 +1096,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector coins_to_uncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { - BOOST_FOREACH(const COutPoint& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -1151,7 +1152,7 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it const Coin& coin = AccessByTxid(*pcoinsTip, hash); - if (!coin.IsPruned()) pindexSlow = chainActive[coin.nHeight]; + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1407,7 +1408,8 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { txundo.vprevout.emplace_back(); - inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); + bool is_spent = inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); + assert(is_spent); } } // add outputs @@ -1450,7 +1452,7 @@ bool CheckTxInputs(const CChainParams& params, const CTransaction& tx, CValidati { const COutPoint &prevout = tx.vin[i].prevout; const Coin& coin = inputs.AccessCoin(prevout); - assert(!coin.IsPruned()); + assert(!coin.IsSpent()); // If prev is coinbase, check that it's matured if (coin.IsCoinBase()) { @@ -1507,7 +1509,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; const Coin& coin = inputs.AccessCoin(prevout); - assert(!coin.IsPruned()); + assert(!coin.IsSpent()); // We very carefully only pass in things to CScriptCheck which // are clearly committed to by tx' witness hash. This provides @@ -1644,14 +1646,14 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - if (view.HaveCoins(out)) fClean = false; // overwriting transaction output + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output if (undo.nHeight == 0) { // Missing undo metadata (height and coinbase). Older versions included this // information only in undo records for the last spend of a transactions' // outputs. This implies that it must be present for some other output of the same tx. const Coin& alternate = AccessByTxid(view, out.hash); - if (!alternate.IsPruned()) { + if (!alternate.IsSpent()) { undo.nHeight = alternate.nHeight; undo.fCoinBase = alternate.fCoinBase; } else { @@ -1693,8 +1695,8 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!tx.vout[o].scriptPubKey.IsUnspendable()) { COutPoint out(hash, o); Coin coin; - view.SpendCoin(out, &coin); - if (tx.vout[o] != coin.out) { + bool is_spent = view.SpendCoin(out, &coin); + if (!is_spent || tx.vout[o] != coin.out) { fClean = false; // transaction output mismatch } } @@ -1914,7 +1916,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fEnforceBIP30) { for (const auto& tx : block.vtx) { for (size_t o = 0; o < tx->vout.size(); o++) { - if (view.HaveCoins(COutPoint(tx->GetHash(), o))) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), REJECT_INVALID, "bad-txns-BIP30"); } @@ -2129,9 +2131,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), - std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. diff --git a/src/validation.h b/src/validation.h index 6bad0e7f489..d06e57a088f 100644 --- a/src/validation.h +++ b/src/validation.h @@ -40,6 +40,7 @@ class CBlockTreeDB; class CBloomFilter; class CBlockUndo; class CChainParams; +class CCoinsViewDB; class CInv; class CConnman; class CScriptCheck; @@ -552,6 +553,9 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex); /** The currently-connected chain of blocks (protected by cs_main). */ extern CChain chainActive; +/** Global variable that points to the coins database (protected by cs_main) */ +extern CCoinsViewDB *pcoinsdbview; + /** Global variable that points to the active CCoinsView (protected by cs_main) */ extern CCoinsViewCache *pcoinsTip; From 2dddcc32129c6014b133ef972a25bf9f5fdfbe8b Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Mar 2019 12:08:32 -0400 Subject: [PATCH 24/32] add unused SnapshotMetadata class (cherry picked from commit 707fde7b9ba522c22179e2db0ed7b462c65138d9) --- src/Makefile.am | 1 + src/node/utxo_snapshot.h | 50 ++++++++++++++++++++++++++++++++++++++++ src/validation.h | 1 + 3 files changed, 52 insertions(+) create mode 100644 src/node/utxo_snapshot.h diff --git a/src/Makefile.am b/src/Makefile.am index d2b7eedfb0a..78168c3f871 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -130,6 +130,7 @@ BITCOIN_CORE_H = \ netaddress.h \ netbase.h \ netmessagemaker.h \ + node/utxo_snapshot.h \ noui.h \ policy/fees.h \ policy/policy.h \ diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h new file mode 100644 index 00000000000..702a0cbe536 --- /dev/null +++ b/src/node/utxo_snapshot.h @@ -0,0 +1,50 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H +#define BITCOIN_NODE_UTXO_SNAPSHOT_H + +#include +#include + +//! Metadata describing a serialized version of a UTXO set from which an +//! assumeutxo CChainState can be constructed. +class SnapshotMetadata +{ +public: + //! The hash of the block that reflects the tip of the chain for the + //! UTXO set contained in this snapshot. + uint256 m_base_blockhash; + + //! The number of coins in the UTXO set contained in this snapshot. Used + //! during snapshot load to estimate progress of UTXO set reconstruction. + uint64_t m_coins_count = 0; + + //! Necessary to "fake" the base nChainTx so that we can estimate progress during + //! initial block download for the assumeutxo chainstate. + unsigned int m_nchaintx = 0; + + SnapshotMetadata() { } + SnapshotMetadata( + const uint256& base_blockhash, + uint64_t coins_count, + unsigned int nchaintx) : + m_base_blockhash(base_blockhash), + m_coins_count(coins_count), + m_nchaintx(nchaintx) { } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(m_base_blockhash); + READWRITE(m_coins_count); + READWRITE(m_nchaintx); + } + +}; + +#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/validation.h b/src/validation.h index d06e57a088f..89272dd7254 100644 --- a/src/validation.h +++ b/src/validation.h @@ -15,6 +15,7 @@ #include "chain.h" #include "coins.h" #include "fs.h" +#include "node/utxo_snapshot.h" #include "policy/policy.h" // For RECOMMENDED_MIN_TX_FEE #include "protocol.h" // For CMessageHeader::MessageStartChars #include "script/script_error.h" From 911c75077242e5741b4771338dc60d4f2705d68e Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Mar 2019 12:09:06 -0400 Subject: [PATCH 25/32] coinstats: add coins_count Also changes existing CCoinsStats attributes to be C++11 initialized. (cherry picked from commit 92fafb3a7da66f737e960e541fcfbcadedf6043a) --- src/rpc/blockchain.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 26c2bb288d6..b97fb1596af 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -860,15 +860,14 @@ UniValue getblock(const JSONRPCRequest& request) struct CCoinsStats { - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint256 hashSerialized; - uint64_t nDiskSize; - arith_uint256 nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nDiskSize(0), nTotalAmount(0) {} + int nHeight{0}; + uint256 hashBlock{}; + uint64_t nTransactions{0}; + uint64_t nTransactionOutputs{0}; + uint256 hashSerialized{}; + uint64_t nDiskSize{0}; + arith_uint256 nTotalAmount{0}; + uint64_t coins_count{0}; }; static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map& outputs) @@ -890,6 +889,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { + stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); @@ -912,6 +912,7 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) } prevkey = key.hash; outputs[key.n] = std::move(coin); + stats.coins_count++; } else { return error("%s: unable to read value", __func__); } From 316c7fc27c6abfb344ed13176d34e721d5ad6cb8 Mon Sep 17 00:00:00 2001 From: James O'Beirne Date: Thu, 28 Mar 2019 12:10:33 -0400 Subject: [PATCH 26/32] rpc: add dumptxoutset Allows the creation of a UTXO snapshot to disk. (cherry picked from commit 57cf74c9918d10c69a46e6ceb3cb1a5e04edf5bc) --- src/rpc/blockchain.cpp | 96 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index b97fb1596af..29ee7fe288a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,6 +14,8 @@ #include "consensus/validation.h" #include "core_io.h" #include "dogecoin.h" +#include "fs.h" +#include "node/utxo_snapshot.h" #include "validation.h" #include "policy/policy.h" #include "primitives/transaction.h" @@ -1867,6 +1869,99 @@ static UniValue getblockstats(const JSONRPCRequest& request) return ret; } +/** + * Serialize the UTXO set to a file for loading elsewhere. + * + * @see SnapshotMetadata + */ +UniValue dumptxoutset(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) + throw runtime_error( + "dumptxoutset \"path\"\n" + "\nWrite the serialized UTXO set to disk.\n" + "Incidentally flushes the latest chainstate database to disk.\n" + "\nArguments:\n" + "1. \"path\" (string, required) Path to the output file. If relative, will be prefixed by datadir.\n" + "\nResult:\n" + "{\n" + " \"coins_written\": n, (numeric) The number of coins written in the snapshot\n" + " \"base_hash\": \"hex\", (string) The hash of the base of the snapshot\n" + " \"base_height\": n, (numeric) The height of the base of the snapshot\n" + " \"path\": \"path\" (string) The absolute path to the snapshot file\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("dumptxoutset", "\"utxo.dat\"") + + HelpExampleRpc("dumptxoutset", "\"utxo.dat\"") + ); + + fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir()); + + if (fs::exists(path) || fs::exists(temppath)) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + path.string() + " or its temporary file already exists. Move it out of the way first"); + } + + FILE* file = fsbridge::fopen(temppath, "wb"); + if (file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't open file " + temppath.string() + " for writing"); + } + + CAutoFile afile(file, SER_DISK, CLIENT_VERSION); + std::unique_ptr pcursor; + CCoinsStats stats; + CBlockIndex* tip; + + { + LOCK(cs_main); + FlushStateToDisk(); + + if (!GetUTXOStats(pcoinsdbview, stats)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + + pcursor = std::unique_ptr(pcoinsdbview->Cursor()); + BlockMap::iterator it = mapBlockIndex.find(stats.hashBlock); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to find UTXO base block"); + } + tip = it->second; + } + + SnapshotMetadata metadata(tip->GetBlockHash(), stats.coins_count, tip->nChainTx); + afile << metadata; + + COutPoint key; + Coin coin; + unsigned int iter = 0; + + while (pcursor->Valid()) { + if (iter % 5000 == 0 && !IsRPCRunning()) { + throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down"); + } + ++iter; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + afile << key; + afile << coin; + } else { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); + } + pcursor->Next(); + } + + afile.fclose(); + fs::rename(temppath, path); + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_written", (int64_t)stats.coins_count); + result.pushKV("base_hash", tip->GetBlockHash().GetHex()); + result.pushKV("base_height", (int64_t)tip->nHeight); + result.pushKV("path", path.string()); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- @@ -1897,6 +1992,7 @@ static const CRPCCommand commands[] = { "hidden", "waitfornewblock", &waitfornewblock, true, {"timeout"} }, { "hidden", "waitforblock", &waitforblock, true, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, true, {"height","timeout"} }, + { "hidden", "dumptxoutset", &dumptxoutset, true, {"path"} }, }; void RegisterBlockchainRPCCommands(CRPCTable &t) From fd5f64370bba68b369de260e7dd00eaf9a42042c Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Mon, 22 Jun 2026 21:23:54 +0000 Subject: [PATCH 27/32] session 4: add dumptxoutset QA test --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/dumptxoutset.py | 84 ++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 qa/rpc-tests/dumptxoutset.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index ad4beae7083..f697b6b4578 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -148,6 +148,7 @@ 'nodehandling.py', 'decodescript.py', 'blockchain.py', + 'dumptxoutset.py', 'disablewallet.py', 'keypool.py', 'p2p-mempool.py', diff --git a/qa/rpc-tests/dumptxoutset.py b/qa/rpc-tests/dumptxoutset.py new file mode 100644 index 00000000000..9bf5fa5ec09 --- /dev/null +++ b/qa/rpc-tests/dumptxoutset.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019-2022 The Bitcoin Core developers +# Copyright (c) 2024 The Dogecoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test the dumptxoutset RPC call. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_jsonrpc, + start_node, +) + + +class DumptxoutsetTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + def setup_network(self, split=False): + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, [])) + self.is_network_split = False + + def run_test(self): + """Test basic dumptxoutset functionality.""" + node = self.nodes[0] + + # Mine some blocks so there are UTXOs to dump + node.generate(110) + + # Determine the output path inside the node's datadir + datadir = os.path.join(self.options.tmpdir, "node0") + out_path = os.path.join(datadir, "utxos.dat") + temp_path = out_path + ".incomplete" + + # Call dumptxoutset + result = node.dumptxoutset(out_path) + + # Validate return fields + assert_equal(result['base_height'], 110) + assert_equal(len(result['base_hash']), 64) + assert_equal(result['path'], out_path) + assert_greater_than(result['coins_written'], 0) + # 110 coinbase transactions = 110 UTXOs (one per block) + assert_equal(result['coins_written'], 110) + + # The final file should exist + assert os.path.exists(out_path), "snapshot file was not created" + + # The temporary (.incomplete) file should have been renamed away + assert not os.path.exists(temp_path), ".incomplete file still present" + + # File must be non-empty + assert_greater_than(os.path.getsize(out_path), 0) + + # Calling again with the same path should fail + assert_raises_jsonrpc( + -8, + "already exists", + node.dumptxoutset, + out_path, + ) + + # Passing a relative path returns an absolute path in the result + result2 = node.dumptxoutset("utxos2.dat") + assert os.path.isabs(result2['path']) + assert result2['path'].endswith("utxos2.dat") + assert os.path.exists(result2['path']) + + print("dumptxoutset tests passed") + + +if __name__ == '__main__': + DumptxoutsetTest().main() From 5e2aa8b46d412024bb62fed507379ed55eeeb05b Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Tue, 23 Jun 2026 18:38:15 +0000 Subject: [PATCH 28/32] session 5: enhance dumptxoutset with snapshot magic/version, txoutset_hash, nchaintx --- qa/rpc-tests/dumptxoutset.py | 3 ++ src/node/utxo_snapshot.h | 55 ++++++++++++++++++++++++++++++++---- src/rpc/blockchain.cpp | 14 +++++---- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/qa/rpc-tests/dumptxoutset.py b/qa/rpc-tests/dumptxoutset.py index 9bf5fa5ec09..794bdbbf8a4 100644 --- a/qa/rpc-tests/dumptxoutset.py +++ b/qa/rpc-tests/dumptxoutset.py @@ -53,6 +53,9 @@ def run_test(self): assert_greater_than(result['coins_written'], 0) # 110 coinbase transactions = 110 UTXOs (one per block) assert_equal(result['coins_written'], 110) + # New fields added in session 5 + assert_equal(len(result['txoutset_hash']), 64) + assert_greater_than(result['nchaintx'], 0) # The final file should exist assert os.path.exists(out_path), "snapshot file was not created" diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 702a0cbe536..42e5be5cc54 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -1,19 +1,36 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2024 The Dogecoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include +#include +#include #include #include +//! Magic bytes written at the start of every UTXO snapshot file. +//! Used to detect files that are not UTXO snapshots. +static const std::array SNAPSHOT_MAGIC_BYTES = {{'u', 't', 'x', 'o', 0xff}}; + +//! Current UTXO snapshot file format version. +static const uint16_t SNAPSHOT_VERSION = 1; + //! Metadata describing a serialized version of a UTXO set from which an //! assumeutxo CChainState can be constructed. +//! All metadata fields come from an untrusted file, so must be validated +//! before being used. class SnapshotMetadata { public: + //! The network magic (pchMessageStart) identifying the chain this snapshot + //! belongs to. Used to detect cross-chain load attempts. + uint8_t m_network_magic[4]; + //! The hash of the block that reflects the tip of the chain for the //! UTXO set contained in this snapshot. uint256 m_base_blockhash; @@ -22,29 +39,57 @@ class SnapshotMetadata //! during snapshot load to estimate progress of UTXO set reconstruction. uint64_t m_coins_count = 0; - //! Necessary to "fake" the base nChainTx so that we can estimate progress during - //! initial block download for the assumeutxo chainstate. + //! The number of transactions in the chain up to and including the base + //! block; used to estimate IBD progress for the loaded chainstate. unsigned int m_nchaintx = 0; - SnapshotMetadata() { } + SnapshotMetadata() { memset(m_network_magic, 0, sizeof(m_network_magic)); } + SnapshotMetadata( + const uint8_t* network_magic, const uint256& base_blockhash, uint64_t coins_count, unsigned int nchaintx) : m_base_blockhash(base_blockhash), m_coins_count(coins_count), - m_nchaintx(nchaintx) { } + m_nchaintx(nchaintx) + { + memcpy(m_network_magic, network_magic, sizeof(m_network_magic)); + } ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream& s, Operation ser_action) { + if (!ser_action.ForRead()) { + // Serialize: write magic bytes and version first + std::array magic = SNAPSHOT_MAGIC_BYTES; + READWRITE(FLATDATA(magic)); + uint16_t version = SNAPSHOT_VERSION; + READWRITE(version); + READWRITE(FLATDATA(m_network_magic)); + } else { + // Deserialize: read and validate magic bytes and version + std::array magic; + READWRITE(FLATDATA(magic)); + if (magic != SNAPSHOT_MAGIC_BYTES) { + throw std::ios_base::failure( + "Invalid UTXO snapshot magic bytes. " + "Is this really a snapshot file?"); + } + uint16_t version; + READWRITE(version); + if (version != SNAPSHOT_VERSION) { + throw std::ios_base::failure( + "Unsupported UTXO snapshot version"); + } + READWRITE(FLATDATA(m_network_magic)); + } READWRITE(m_base_blockhash); READWRITE(m_coins_count); READWRITE(m_nchaintx); } - }; #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 29ee7fe288a..03bb497afd1 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1885,10 +1885,12 @@ UniValue dumptxoutset(const JSONRPCRequest& request) "1. \"path\" (string, required) Path to the output file. If relative, will be prefixed by datadir.\n" "\nResult:\n" "{\n" - " \"coins_written\": n, (numeric) The number of coins written in the snapshot\n" - " \"base_hash\": \"hex\", (string) The hash of the base of the snapshot\n" - " \"base_height\": n, (numeric) The height of the base of the snapshot\n" - " \"path\": \"path\" (string) The absolute path to the snapshot file\n" + " \"coins_written\": n, (numeric) The number of coins written in the snapshot\n" + " \"base_hash\": \"hex\", (string) The hash of the base of the snapshot\n" + " \"base_height\": n, (numeric) The height of the base of the snapshot\n" + " \"path\": \"path\", (string) The absolute path to the snapshot file\n" + " \"txoutset_hash\": \"hex\", (string) The hash of the UTXO set contents\n" + " \"nchaintx\": n (numeric) The number of transactions in the chain up to and including the base block\n" "}\n" "\nExamples:\n" + HelpExampleCli("dumptxoutset", "\"utxo.dat\"") @@ -1930,7 +1932,7 @@ UniValue dumptxoutset(const JSONRPCRequest& request) tip = it->second; } - SnapshotMetadata metadata(tip->GetBlockHash(), stats.coins_count, tip->nChainTx); + SnapshotMetadata metadata(Params().MessageStart(), tip->GetBlockHash(), stats.coins_count, tip->nChainTx); afile << metadata; COutPoint key; @@ -1959,6 +1961,8 @@ UniValue dumptxoutset(const JSONRPCRequest& request) result.pushKV("base_hash", tip->GetBlockHash().GetHex()); result.pushKV("base_height", (int64_t)tip->nHeight); result.pushKV("path", path.string()); + result.pushKV("txoutset_hash", stats.hashSerialized.GetHex()); + result.pushKV("nchaintx", (int64_t)tip->nChainTx); return result; } From 898964758396075dfdd3ebe4e9f75c321bc538fb Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Tue, 23 Jun 2026 18:58:10 +0000 Subject: [PATCH 29/32] session 6: add adapted loadtxoutset RPC and test scaffolding --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/loadtxoutset.py | 72 +++++++++++++ src/rpc/blockchain.cpp | 202 +++++++++++++++++++++++++++++++++++ src/validation.cpp | 59 ++++++++++ src/validation.h | 6 ++ 5 files changed, 340 insertions(+) create mode 100644 qa/rpc-tests/loadtxoutset.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index f697b6b4578..0df557b08a8 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -149,6 +149,7 @@ 'decodescript.py', 'blockchain.py', 'dumptxoutset.py', + 'loadtxoutset.py', 'disablewallet.py', 'keypool.py', 'p2p-mempool.py', diff --git a/qa/rpc-tests/loadtxoutset.py b/qa/rpc-tests/loadtxoutset.py new file mode 100644 index 00000000000..5248ad3a67d --- /dev/null +++ b/qa/rpc-tests/loadtxoutset.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 The Dogecoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Test the loadtxoutset RPC call. +""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_jsonrpc, + start_node, +) + + +class LoadtxoutsetTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + + def setup_network(self, split=False): + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, [])) + self.is_network_split = False + + def run_test(self): + node = self.nodes[0] + + node.generate(110) + datadir = os.path.join(self.options.tmpdir, "node0") + snapshot_path = os.path.join(datadir, "utxos-load.dat") + + dump_result = node.dumptxoutset(snapshot_path) + snapshot_info = node.gettxoutsetinfo() + + assert_equal(dump_result["base_height"], 110) + assert_equal(snapshot_info["hash_serialized_2"], dump_result["txoutset_hash"]) + + node.generate(5) + assert_equal(node.getblockcount(), 115) + + assert_raises_jsonrpc( + -8, + "Snapshot content hash mismatch", + node.loadtxoutset, + snapshot_path, + "00" * 32, + ) + + load_result = node.loadtxoutset(snapshot_path, dump_result["txoutset_hash"]) + + assert_equal(load_result["coins_loaded"], dump_result["coins_written"]) + assert_equal(load_result["base_height"], dump_result["base_height"]) + assert_equal(load_result["base_hash"], dump_result["base_hash"]) + assert_equal(load_result["path"], snapshot_path) + + restored_info = node.gettxoutsetinfo() + assert_equal(node.getblockcount(), 110) + assert_equal(restored_info["bestblock"], snapshot_info["bestblock"]) + assert_equal(restored_info["transactions"], snapshot_info["transactions"]) + assert_equal(restored_info["txouts"], snapshot_info["txouts"]) + assert_equal(restored_info["hash_serialized_2"], snapshot_info["hash_serialized_2"]) + + +if __name__ == '__main__': + LoadtxoutsetTest().main() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 03bb497afd1..2d2fa00285e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -928,6 +928,98 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) return true; } +static bool ComputeSnapshotStats( + CAutoFile& afile, + const SnapshotMetadata& metadata, + CCoinsStats& stats, + std::string& error) +{ + stats = CCoinsStats(); + stats.hashBlock = metadata.m_base_blockhash; + + { + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(metadata.m_base_blockhash); + if (it == mapBlockIndex.end()) { + error = "Unable to find snapshot base block"; + return false; + } + stats.nHeight = it->second->nHeight; + } + + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + ss << stats.hashBlock; + + uint256 prevkey; + std::map outputs; + + for (uint64_t coins_loaded = 0; coins_loaded < metadata.m_coins_count; ++coins_loaded) { + try { + COutPoint key; + Coin coin; + afile >> key; + afile >> coin; + + if (coin.nHeight > static_cast(stats.nHeight)) { + error = strprintf("Bad snapshot data after deserializing %d coins", coins_loaded); + return false; + } + if (!MoneyRange(coin.out.nValue)) { + error = strprintf("Bad snapshot data after deserializing %d coins - bad tx out value", coins_loaded); + return false; + } + + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); + } + prevkey = key.hash; + outputs[key.n] = coin; + stats.coins_count++; + } catch (const std::ios_base::failure&) { + error = strprintf("Bad snapshot format or truncated snapshot after deserializing %d coins", coins_loaded); + return false; + } + } + + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } + stats.hashSerialized = ss.GetHash(); + + try { + unsigned char leftover; + afile >> leftover; + error = strprintf("Bad snapshot - coins left over after deserializing %d coins", metadata.m_coins_count); + return false; + } catch (const std::ios_base::failure&) { + } + + return true; +} + +static bool LoadSnapshotCoins(CAutoFile& afile, const SnapshotMetadata& metadata) +{ + CCoinsViewCache snapshot_cache(pcoinsdbview); + + for (uint64_t coins_loaded = 0; coins_loaded < metadata.m_coins_count; ++coins_loaded) { + COutPoint key; + Coin coin; + afile >> key; + afile >> coin; + snapshot_cache.AddCoin(key, std::move(coin), false); + + if ((coins_loaded + 1) % 50000 == 0) { + if (!snapshot_cache.Flush()) { + return false; + } + } + } + + snapshot_cache.SetBestBlock(metadata.m_base_blockhash); + return snapshot_cache.Flush(); +} + UniValue pruneblockchain(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) @@ -1966,6 +2058,115 @@ UniValue dumptxoutset(const JSONRPCRequest& request) return result; } +UniValue loadtxoutset(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 2) + throw runtime_error( + "loadtxoutset \"path\" \"txoutset_hash\"\n" + "\nLoad a previously dumped UTXO set from disk into the active chainstate.\n" + "The snapshot base block must already be part of the active chain, and the\n" + "provided txoutset hash must match the snapshot contents.\n" + "\nArguments:\n" + "1. \"path\" (string, required) Path to the snapshot file. If relative, will be prefixed by datadir.\n" + "2. \"txoutset_hash\" (string, required) Expected hash_serialized_2 for the snapshot contents.\n" + "\nResult:\n" + "{\n" + " \"coins_loaded\": n, (numeric) The number of coins loaded from the snapshot\n" + " \"base_hash\": \"hex\", (string) The hash of the base of the snapshot\n" + " \"base_height\": n, (numeric) The height of the base of the snapshot\n" + " \"path\": \"path\" (string) The absolute path to the snapshot file\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("loadtxoutset", "\"utxo.dat\" \"0123456789abcdef\"") + + HelpExampleRpc("loadtxoutset", "\"utxo.dat\", \"0123456789abcdef\"") + ); + + const fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir()); + const uint256 expected_hash = ParseHashV(request.params[1], "txoutset_hash"); + + FILE* file = fsbridge::fopen(path, "rb"); + if (file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't open file " + path.string() + " for reading"); + } + + SnapshotMetadata metadata; + { + CAutoFile afile(file, SER_DISK, CLIENT_VERSION); + try { + afile >> metadata; + } catch (const std::ios_base::failure& e) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Unable to parse metadata: %s", e.what())); + } + } + + if (memcmp(metadata.m_network_magic, Params().MessageStart(), sizeof(metadata.m_network_magic)) != 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Snapshot network does not match this node"); + } + + CBlockIndex* snapshot_index = NULL; + { + LOCK(cs_main); + BlockMap::iterator it = mapBlockIndex.find(metadata.m_base_blockhash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Snapshot base block header not found"); + } + snapshot_index = it->second; + if (!chainActive.Contains(snapshot_index)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Snapshot base block must already be part of the active chain"); + } + } + + CCoinsStats snapshot_stats; + std::string snapshot_error; + { + FILE* verify_file = fsbridge::fopen(path, "rb"); + if (verify_file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't reopen file " + path.string() + " for reading"); + } + CAutoFile afile(verify_file, SER_DISK, CLIENT_VERSION); + afile >> metadata; + if (!ComputeSnapshotStats(afile, metadata, snapshot_stats, snapshot_error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, snapshot_error); + } + } + + if (snapshot_stats.hashSerialized != expected_hash) { + throw JSONRPCError( + RPC_INVALID_PARAMETER, + strprintf("Snapshot content hash mismatch: expected %s, got %s", + expected_hash.GetHex(), + snapshot_stats.hashSerialized.GetHex())); + } + + CValidationState state; + if (!RewindChainstateToGenesis(state, Params())) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to rewind chainstate: " + FormatStateMessage(state)); + } + + { + FILE* load_file = fsbridge::fopen(path, "rb"); + if (load_file == nullptr) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Couldn't reopen file " + path.string() + " for reading"); + } + CAutoFile afile(load_file, SER_DISK, CLIENT_VERSION); + afile >> metadata; + if (!LoadSnapshotCoins(afile, metadata)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to load snapshot coins into chainstate"); + } + } + + if (!ActivateSnapshotTip(state, Params(), snapshot_index)) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to activate loaded snapshot: " + FormatStateMessage(state)); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("coins_loaded", (int64_t)metadata.m_coins_count); + result.pushKV("base_hash", snapshot_index->GetBlockHash().GetHex()); + result.pushKV("base_height", (int64_t)snapshot_index->nHeight); + result.pushKV("path", path.string()); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- @@ -1997,6 +2198,7 @@ static const CRPCCommand commands[] = { "hidden", "waitforblock", &waitforblock, true, {"blockhash","timeout"} }, { "hidden", "waitforblockheight", &waitforblockheight, true, {"height","timeout"} }, { "hidden", "dumptxoutset", &dumptxoutset, true, {"path"} }, + { "hidden", "loadtxoutset", &loadtxoutset, true, {"path", "txoutset_hash"} }, }; void RegisterBlockchainRPCCommands(CRPCTable &t) diff --git a/src/validation.cpp b/src/validation.cpp index 15603ab4551..039b5f8b0bd 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2755,6 +2755,65 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C return true; } +bool RewindChainstateToGenesis(CValidationState& state, const CChainParams& chainparams) +{ + LOCK(cs_main); + + if (mempool.size() != 0) { + return state.Error("mempool not empty"); + } + + for (CBlockIndex* pindex = chainActive.Tip(); pindex != NULL && pindex->pprev != NULL; pindex = pindex->pprev) { + if ((pindex->nStatus & BLOCK_HAVE_MASK) != BLOCK_HAVE_MASK) { + return state.Error(strprintf("snapshot restore requires block and undo data for block %s", pindex->GetBlockHash().ToString())); + } + } + + while (chainActive.Height() > 0) { + if (!DisconnectTip(state, chainparams, true)) { + return error("RewindChainstateToGenesis: unable to disconnect block at height %i", chainActive.Height()); + } + } + + return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); +} + +bool ActivateSnapshotTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexSnapshot) +{ + CBlockIndex* pindexFork = NULL; + bool fInitialDownload = false; + + { + LOCK(cs_main); + + if (pindexSnapshot == NULL) { + return state.Error("snapshot base block not found"); + } + if (chainActive.Height() != 0) { + return state.Error("chainstate must be rewound to genesis before activating snapshot"); + } + if (!pindexSnapshot->IsValid(BLOCK_VALID_SCRIPTS) || pindexSnapshot->nChainTx == 0) { + return state.Error("snapshot base block must already be fully validated"); + } + + pindexFork = chainActive.Tip(); + pcoinsTip->SetBestBlock(pindexSnapshot->GetBlockHash()); + UpdateTip(pindexSnapshot, chainparams); + setBlockIndexCandidates.insert(pindexSnapshot); + PruneBlockIndexCandidates(); + CheckBlockIndex(chainparams.GetConsensus(chainActive.Height())); + fInitialDownload = IsInitialBlockDownload(); + + if (!FlushStateToDisk(state, FLUSH_STATE_ALWAYS)) { + return false; + } + } + + GetMainSignals().UpdatedBlockTip(pindexSnapshot, pindexFork, fInitialDownload); + uiInterface.NotifyBlockTip(fInitialDownload, pindexSnapshot); + return true; +} + bool ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); diff --git a/src/validation.h b/src/validation.h index 89272dd7254..63283ebdad6 100644 --- a/src/validation.h +++ b/src/validation.h @@ -548,6 +548,12 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn /** Mark a block as invalid. */ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex); +/** Rewind the active chainstate back to genesis. */ +bool RewindChainstateToGenesis(CValidationState& state, const CChainParams& chainparams); + +/** Set the active chain tip to a block whose chainstate has been loaded separately. */ +bool ActivateSnapshotTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexSnapshot); + /** Remove invalidity status from a block and its descendants. */ bool ResetBlockFailureFlags(CBlockIndex *pindex); From 2f4066c811a69b501f69f001af4ed1af1b272bed Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Tue, 23 Jun 2026 19:29:17 +0000 Subject: [PATCH 30/32] session 7: fix setBlockIndexCandidates invariant in RewindChainstateToGenesis --- src/validation.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/validation.cpp b/src/validation.cpp index 039b5f8b0bd..9c3f6e210bf 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2770,9 +2770,15 @@ bool RewindChainstateToGenesis(CValidationState& state, const CChainParams& chai } while (chainActive.Height() > 0) { + CBlockIndex* pindexDisconnecting = chainActive.Tip(); if (!DisconnectTip(state, chainparams, true)) { return error("RewindChainstateToGenesis: unable to disconnect block at height %i", chainActive.Height()); } + // Re-add disconnected blocks to the candidate set so that CheckBlockIndex + // invariants are satisfied after ActivateSnapshotTip sets a new tip. + // This mirrors what ActivateBestChainStep does when disconnecting blocks + // during a reorg. + setBlockIndexCandidates.insert(pindexDisconnecting); } return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); From b91c6835c5583a6b92c6252fc3daa8937bcd1a9c Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Tue, 23 Jun 2026 20:07:52 +0000 Subject: [PATCH 31/32] session 8: add snapshot RPC QA and docs --- doc/rpc-maturity.md | 2 ++ qa/rpc-tests/dumptxoutset.py | 9 +++++++++ qa/rpc-tests/loadtxoutset.py | 38 +++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/doc/rpc-maturity.md b/doc/rpc-maturity.md index d71d38959c5..7fd873dc3e3 100644 --- a/doc/rpc-maturity.md +++ b/doc/rpc-maturity.md @@ -28,6 +28,7 @@ Core RPC. Maturity is expressed over 3 stages: | decoderawtransaction | STABLE | | | decodescript | STABLE | | | disconnectnode | STABLE | | +| dumptxoutset | UNSTABLE | New in 1.14.99 | | dumpprivkey | STABLE | | | dumpwallet | UNSTABLE | Breaking change since 1.14.6 | | encryptwallet | STABLE | | @@ -97,6 +98,7 @@ Core RPC. Maturity is expressed over 3 stages: | listtransactions | STABLE | | | listunspent | STABLE | | | lockunspent | STABLE | | +| loadtxoutset | UNSTABLE | New in 1.14.99 | | move | DEPRECATED | Deprecated since 1.14.0 | | ping | STABLE | | | preciousblock | STABLE | | diff --git a/qa/rpc-tests/dumptxoutset.py b/qa/rpc-tests/dumptxoutset.py index 794bdbbf8a4..1493a25b7bd 100644 --- a/qa/rpc-tests/dumptxoutset.py +++ b/qa/rpc-tests/dumptxoutset.py @@ -80,6 +80,15 @@ def run_test(self): assert result2['path'].endswith("utxos2.dat") assert os.path.exists(result2['path']) + # Invalid output directories should fail cleanly + invalid_path = os.path.join(datadir, "missing", "utxos3.dat") + assert_raises_jsonrpc( + -8, + "Couldn't open file", + node.dumptxoutset, + invalid_path, + ) + print("dumptxoutset tests passed") diff --git a/qa/rpc-tests/loadtxoutset.py b/qa/rpc-tests/loadtxoutset.py index 5248ad3a67d..5d8895dd3d2 100644 --- a/qa/rpc-tests/loadtxoutset.py +++ b/qa/rpc-tests/loadtxoutset.py @@ -36,12 +36,49 @@ def run_test(self): datadir = os.path.join(self.options.tmpdir, "node0") snapshot_path = os.path.join(datadir, "utxos-load.dat") + missing_snapshot = os.path.join(datadir, "does-not-exist.dat") + assert_raises_jsonrpc( + -8, + "Couldn't open file", + node.loadtxoutset, + missing_snapshot, + "00" * 32, + ) + dump_result = node.dumptxoutset(snapshot_path) snapshot_info = node.gettxoutsetinfo() assert_equal(dump_result["base_height"], 110) assert_equal(snapshot_info["hash_serialized_2"], dump_result["txoutset_hash"]) + bad_magic_path = snapshot_path + ".badmagic" + with open(snapshot_path, "rb") as snapshot_file: + snapshot_bytes = bytearray(snapshot_file.read()) + snapshot_bytes[0] ^= 0x01 + with open(bad_magic_path, "wb") as snapshot_file: + snapshot_file.write(snapshot_bytes) + assert_raises_jsonrpc( + -22, + "Unable to parse metadata: Invalid UTXO snapshot magic bytes", + node.loadtxoutset, + bad_magic_path, + dump_result["txoutset_hash"], + ) + + bad_network_path = snapshot_path + ".badnet" + with open(snapshot_path, "rb") as snapshot_file: + snapshot_bytes = bytearray(snapshot_file.read()) + snapshot_bytes[7:11] = b"\x00\x00\x00\x00" + with open(bad_network_path, "wb") as snapshot_file: + snapshot_file.write(snapshot_bytes) + assert_raises_jsonrpc( + -8, + "Snapshot network does not match this node", + node.loadtxoutset, + bad_network_path, + dump_result["txoutset_hash"], + ) + node.generate(5) assert_equal(node.getblockcount(), 115) @@ -67,6 +104,5 @@ def run_test(self): assert_equal(restored_info["txouts"], snapshot_info["txouts"]) assert_equal(restored_info["hash_serialized_2"], snapshot_info["hash_serialized_2"]) - if __name__ == '__main__': LoadtxoutsetTest().main() From 7c79b9ee2d94e7a313e577bbab77be587b6fb81e Mon Sep 17 00:00:00 2001 From: Ed Tubbs Date: Wed, 24 Jun 2026 16:14:13 +0000 Subject: [PATCH 32/32] Remove invalid PoW check in LoadBlockIndexGuts (scrypt/auxpow) --- src/txdb.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 88494b762c5..31fd6ba4fc1 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -236,8 +236,10 @@ bool CBlockTreeDB::LoadBlockIndexGuts(std::functionnStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; - if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, Params().GetConsensus(pindexNew->nHeight))) - return error("LoadBlockIndex(): CheckProofOfWork failed: %s", pindexNew->ToString()); + /* Bitcoin checks the PoW here. We don't do this because + the CDiskBlockIndex does not contain the auxpow. + This check isn't important, since the data on disk should + already be valid and can be trusted. */ pcursor->Next(); } else {