backport: dumptxoutset and loadtxoutset RPCs

doc/rpc-maturity.md

@@ -28,6 +28,7 @@ Core RPC. Maturity is expressed over 3 stages:
 | decoderawtransaction   | STABLE     |                                            |
 | decodescript           | STABLE     |                                            |
 | disconnectnode         | STABLE     |                                            |
+| dumptxoutset           | UNSTABLE   | New in 1.14.99                             |
 | dumpprivkey            | STABLE     |                                            |
 | dumpwallet             | UNSTABLE   | Breaking change since 1.14.6               |
 | encryptwallet          | STABLE     |                                            |
@@ -97,6 +98,7 @@ Core RPC. Maturity is expressed over 3 stages:
 | listtransactions       | STABLE     |                                            |
 | listunspent            | STABLE     |                                            |
 | lockunspent            | STABLE     |                                            |
+| loadtxoutset           | UNSTABLE   | New in 1.14.99                             |
 | move                   | DEPRECATED | Deprecated since 1.14.0                    |
 | ping                   | STABLE     |                                            |
 | preciousblock          | STABLE     |                                            |

qa/pull-tester/rpc-tests.py

@@ -148,6 +148,8 @@
     'nodehandling.py',
     'decodescript.py',
     'blockchain.py',
+    'dumptxoutset.py',
+    'loadtxoutset.py',
     'disablewallet.py',
     'keypool.py',
     'p2p-mempool.py',

qa/rpc-tests/blockchain.py

@@ -108,9 +108,9 @@ def _test_gettxoutsetinfo(self):
         assert_equal(res['transactions'], 120)
         assert_equal(res['height'], 120)
         assert_equal(res['txouts'], 120)
-        assert_equal(res['bytes_serialized'], 8520),
+        assert_greater_than(res['disk_size'], 0)
         assert_equal(len(res['bestblock']), 64)
-        assert_equal(len(res['hash_serialized']), 64)
+        assert_equal(len(res['hash_serialized_2']), 64)
 
     def _test_getblockheader(self):
         node = self.nodes[0]

qa/rpc-tests/dumptxoutset.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019-2022 The Bitcoin Core developers
+# Copyright (c) 2024 The Dogecoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""
+Test the dumptxoutset RPC call.
+"""
+
+import os
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_greater_than,
+    assert_raises_jsonrpc,
+    start_node,
+)
+
+
+class DumptxoutsetTest(BitcoinTestFramework):
+
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def setup_network(self, split=False):
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, []))
+        self.is_network_split = False
+
+    def run_test(self):
+        """Test basic dumptxoutset functionality."""
+        node = self.nodes[0]
+
+        # Mine some blocks so there are UTXOs to dump
+        node.generate(110)
+
+        # Determine the output path inside the node's datadir
+        datadir = os.path.join(self.options.tmpdir, "node0")
+        out_path = os.path.join(datadir, "utxos.dat")
+        temp_path = out_path + ".incomplete"
+
+        # Call dumptxoutset
+        result = node.dumptxoutset(out_path)
+
+        # Validate return fields
+        assert_equal(result['base_height'], 110)
+        assert_equal(len(result['base_hash']), 64)
+        assert_equal(result['path'], out_path)
+        assert_greater_than(result['coins_written'], 0)
+        # 110 coinbase transactions = 110 UTXOs (one per block)
+        assert_equal(result['coins_written'], 110)
+        # New fields added in session 5
+        assert_equal(len(result['txoutset_hash']), 64)
+        assert_greater_than(result['nchaintx'], 0)
+
+        # The final file should exist
+        assert os.path.exists(out_path), "snapshot file was not created"
+
+        # The temporary (.incomplete) file should have been renamed away
+        assert not os.path.exists(temp_path), ".incomplete file still present"
+
+        # File must be non-empty
+        assert_greater_than(os.path.getsize(out_path), 0)
+
+        # Calling again with the same path should fail
+        assert_raises_jsonrpc(
+            -8,
+            "already exists",
+            node.dumptxoutset,
+            out_path,
+        )
+
+        # Passing a relative path returns an absolute path in the result
+        result2 = node.dumptxoutset("utxos2.dat")
+        assert os.path.isabs(result2['path'])
+        assert result2['path'].endswith("utxos2.dat")
+        assert os.path.exists(result2['path'])
+
+        # Invalid output directories should fail cleanly
+        invalid_path = os.path.join(datadir, "missing", "utxos3.dat")
+        assert_raises_jsonrpc(
+            -8,
+            "Couldn't open file",
+            node.dumptxoutset,
+            invalid_path,
+        )
+
+        print("dumptxoutset tests passed")
+
+
+if __name__ == '__main__':
+    DumptxoutsetTest().main()

qa/rpc-tests/loadtxoutset.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 The Dogecoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""
+Test the loadtxoutset RPC call.
+"""
+
+import os
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_raises_jsonrpc,
+    start_node,
+)
+
+
+class LoadtxoutsetTest(BitcoinTestFramework):
+
+    def __init__(self):
+        super().__init__()
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def setup_network(self, split=False):
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, []))
+        self.is_network_split = False
+
+    def run_test(self):
+        node = self.nodes[0]
+
+        node.generate(110)
+        datadir = os.path.join(self.options.tmpdir, "node0")
+        snapshot_path = os.path.join(datadir, "utxos-load.dat")
+
+        missing_snapshot = os.path.join(datadir, "does-not-exist.dat")
+        assert_raises_jsonrpc(
+            -8,
+            "Couldn't open file",
+            node.loadtxoutset,
+            missing_snapshot,
+            "00" * 32,
+        )
+
+        dump_result = node.dumptxoutset(snapshot_path)
+        snapshot_info = node.gettxoutsetinfo()
+
+        assert_equal(dump_result["base_height"], 110)
+        assert_equal(snapshot_info["hash_serialized_2"], dump_result["txoutset_hash"])
+
+        bad_magic_path = snapshot_path + ".badmagic"
+        with open(snapshot_path, "rb") as snapshot_file:
+            snapshot_bytes = bytearray(snapshot_file.read())
+        snapshot_bytes[0] ^= 0x01
+        with open(bad_magic_path, "wb") as snapshot_file:
+            snapshot_file.write(snapshot_bytes)
+        assert_raises_jsonrpc(
+            -22,
+            "Unable to parse metadata: Invalid UTXO snapshot magic bytes",
+            node.loadtxoutset,
+            bad_magic_path,
+            dump_result["txoutset_hash"],
+        )
+
+        bad_network_path = snapshot_path + ".badnet"
+        with open(snapshot_path, "rb") as snapshot_file:
+            snapshot_bytes = bytearray(snapshot_file.read())
+        snapshot_bytes[7:11] = b"\x00\x00\x00\x00"
+        with open(bad_network_path, "wb") as snapshot_file:
+            snapshot_file.write(snapshot_bytes)
+        assert_raises_jsonrpc(
+            -8,
+            "Snapshot network does not match this node",
+            node.loadtxoutset,
+            bad_network_path,
+            dump_result["txoutset_hash"],
+        )
+
+        node.generate(5)
+        assert_equal(node.getblockcount(), 115)
+
+        assert_raises_jsonrpc(
+            -8,
+            "Snapshot content hash mismatch",
+            node.loadtxoutset,
+            snapshot_path,
+            "00" * 32,
+        )
+
+        load_result = node.loadtxoutset(snapshot_path, dump_result["txoutset_hash"])
+
+        assert_equal(load_result["coins_loaded"], dump_result["coins_written"])
+        assert_equal(load_result["base_height"], dump_result["base_height"])
+        assert_equal(load_result["base_hash"], dump_result["base_hash"])
+        assert_equal(load_result["path"], snapshot_path)
+
+        restored_info = node.gettxoutsetinfo()
+        assert_equal(node.getblockcount(), 110)
+        assert_equal(restored_info["bestblock"], snapshot_info["bestblock"])
+        assert_equal(restored_info["transactions"], snapshot_info["transactions"])
+        assert_equal(restored_info["txouts"], snapshot_info["txouts"])
+        assert_equal(restored_info["hash_serialized_2"], snapshot_info["hash_serialized_2"])
+
+if __name__ == '__main__':
+    LoadtxoutsetTest().main()

src/Makefile.am

@@ -130,6 +130,7 @@ BITCOIN_CORE_H = \
   netaddress.h \
   netbase.h \
   netmessagemaker.h \
+  node/utxo_snapshot.h \
   noui.h \
   policy/fees.h \
   policy/policy.h \

src/bench/ccoins_caching.cpp

@@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet)
     dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG;
     dummyTransactions[0].vout[1].nValue = 50 * CENT;
     dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG;
-    coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0);
+    AddCoins(coinsRet, dummyTransactions[0], 0);
 
     dummyTransactions[1].vout.resize(2);
     dummyTransactions[1].vout[0].nValue = 21 * CENT;
     dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID());
     dummyTransactions[1].vout[1].nValue = 22 * CENT;
     dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID());
-    coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0);
+    AddCoins(coinsRet, dummyTransactions[1], 0);
 
     return dummyTransactions;
 }

src/bitcoin-tx.cpp

@@ -576,24 +576,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr)
             if (nOut < 0)
                 throw std::runtime_error("vout must be positive");
 
+            COutPoint out(txid, nOut);
             std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey"));
             CScript scriptPubKey(pkData.begin(), pkData.end());
 
             {
-                CCoinsModifier coins = view.ModifyCoins(txid);
-                if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) {
+                const Coin& coin = view.AccessCoin(out);
+                if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) {
                     std::string err("Previous output scriptPubKey mismatch:\n");
-                    err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+
+                    err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+
                         ScriptToAsmStr(scriptPubKey);
                     throw std::runtime_error(err);
                 }
-                if ((unsigned int)nOut >= coins->vout.size())
-                    coins->vout.resize(nOut+1);
-                coins->vout[nOut].scriptPubKey = scriptPubKey;
-                coins->vout[nOut].nValue = 0;
+                Coin newcoin;
+                newcoin.out.scriptPubKey = scriptPubKey;
+                newcoin.out.nValue = 0;
                 if (prevOut.exists("amount")) {
-                    coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]);
+                    newcoin.out.nValue = AmountFromValue(prevOut["amount"]);
                 }
+                newcoin.nHeight = 1;
+                view.AddCoin(out, std::move(newcoin), true);
             }
 
             // if redeemScript given and private keys given,
@@ -615,13 +617,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr)
     // Sign what we can:
     for (unsigned int i = 0; i < mergedTx.vin.size(); i++) {
         CTxIn& txin = mergedTx.vin[i];
-        const CCoins* coins = view.AccessCoins(txin.prevout.hash);
-        if (!coins || !coins->IsAvailable(txin.prevout.n)) {
+        const Coin& coin = view.AccessCoin(txin.prevout);
+        if (coin.IsSpent()) {
             fComplete = false;
             continue;
         }
-        const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey;
-        const CAmount& amount = coins->vout[txin.prevout.n].nValue;
+        const CScript& prevPubKey = coin.out.scriptPubKey;
+        const CAmount& amount = coin.out.nValue;
 
         SignatureData sigdata;
         // Only sign SIGHASH_SINGLE if there's a corresponding output: