#!/usr/bin/env python3 # Copyright (c) 2014-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. By default, bitcoind will dump mempool on shutdown and then reload it on startup. This can be overridden with the -persistmempool=0 command line option. Test is as follows: - start node0, node1 and node2. node1 has -persistmempool=0 - create 5 transactions on node2 to its own address. Note that these are not sent to node0 or node1 addresses because we don't want them to be saved in the wallet. - check that node0 and node1 have 5 transactions in their mempools - shutdown all nodes. - startup node0. Verify that it still has 5 transactions in its mempool. Shutdown node0. This tests that by default the mempool is persistent. - startup node1. Verify that its mempool is empty. Shutdown node1. This tests that with -persistmempool=0, the mempool is not dumped to disk when the node is shut down. - Restart node0 with -persistmempool=0. Verify that its mempool is empty. Shutdown node0. This tests that with -persistmempool=0, the mempool is not loaded from disk on start up. - Restart node0 with -persistmempool. Verify that it has 5 transactions in its mempool. This tests that -persistmempool=0 does not overwrite a previously valid mempool stored on disk. - Remove node0 mempool.dat and verify savemempool RPC recreates it and verify that node1 can load it and has 5 transactions in its mempool. - Verify that savemempool throws when the RPC is called if node1 can't write to disk. """ from decimal import Decimal import os import time from test_framework.p2p import P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_greater_than_or_equal, assert_raises_rpc_error, ) from test_framework.wallet import MiniWallet, COIN class MempoolPersistTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser, legacy=False) def set_test_params(self): self.num_nodes = 3 self.extra_args = [[], ["-persistmempool=0"], []] def run_test(self): self.mini_wallet = MiniWallet(self.nodes[2]) if self.is_sqlite_compiled(): self.nodes[2].createwallet( wallet_name="watch", descriptors=True, disable_private_keys=True, load_on_startup=False, ) wallet_watch = self.nodes[2].get_wallet_rpc("watch") assert_equal([{'success': True}], wallet_watch.importdescriptors([{'desc': self.mini_wallet.get_descriptor(), 'timestamp': 0}])) self.log.debug("Send 5 transactions from node2 (to its own address)") tx_creation_time_lower = int(time.time()) for _ in range(5): last_txid = self.mini_wallet.send_self_transfer(from_node=self.nodes[2])["txid"] if self.is_sqlite_compiled(): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet node2_balance = wallet_watch.getbalance() self.sync_all() tx_creation_time_higher = int(time.time()) self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools") assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) total_fee_old = self.nodes[0].getmempoolinfo()['total_fee'] self.log.debug("Prioritize a transaction on node0") fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'], fees['modified']) self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000) fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) self.log.info('Check the total base fee is unchanged after prioritisetransaction') assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee']) assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items())) last_entry = self.nodes[0].getmempoolentry(txid=last_txid) tx_creation_time = last_entry['time'] assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) # disconnect nodes & make a txn that remains in the unbroadcast set. self.disconnect_nodes(0, 1) assert_equal(len(self.nodes[0].getpeerinfo()), 0) assert_equal(len(self.nodes[0].p2ps), 0) self.mini_wallet.send_self_transfer(from_node=self.nodes[0]) # Test persistence of prioritisation for transactions not in the mempool. # Create a tx and prioritise but don't submit until after the restart. tx_prioritised_not_submitted = self.mini_wallet.create_self_transfer() self.nodes[0].prioritisetransaction(txid=tx_prioritised_not_submitted['txid'], fee_delta=9999) self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later # Also don't store the mempool, to keep the datadir clean self.start_node(1, extra_args=["-persistmempool=0"]) self.start_node(0) self.start_node(2) assert self.nodes[0].getmempoolinfo()["loaded"] # start_node is blocking on the mempool being loaded assert self.nodes[2].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 6) assert_equal(len(self.nodes[2].getrawmempool()), 5) # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) self.log.debug('Verify prioritization is loaded correctly') fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) self.log.debug('Verify all fields are loaded correctly') assert_equal(last_entry, self.nodes[0].getmempoolentry(txid=last_txid)) self.nodes[0].sendrawtransaction(tx_prioritised_not_submitted['hex']) entry_prioritised_before_restart = self.nodes[0].getmempoolentry(txid=tx_prioritised_not_submitted['txid']) assert_equal(entry_prioritised_before_restart['fees']['base'] + Decimal('0.00009999'), entry_prioritised_before_restart['fees']['modified']) # Verify accounting of mempool transactions after restart is correct if self.is_sqlite_compiled(): self.nodes[2].loadwallet("watch") wallet_watch = self.nodes[2].get_wallet_rpc("watch") self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, wallet_watch.getbalance()) mempooldat0 = os.path.join(self.nodes[0].chain_path, 'mempool.dat') mempooldat1 = os.path.join(self.nodes[1].chain_path, 'mempool.dat') self.log.debug("Force -persistmempool=0 node1 to savemempool to disk via RPC") assert not os.path.exists(mempooldat1) result1 = self.nodes[1].savemempool() assert os.path.isfile(mempooldat1) assert_equal(result1['filename'], mempooldat1) os.remove(mempooldat1) self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 0) self.log.debug("Import mempool at runtime to node0.") assert_equal({}, self.nodes[0].importmempool(mempooldat0)) assert_equal(len(self.nodes[0].getrawmempool()), 7) fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"] assert_equal(fees["base"], fees["modified"]) assert_equal({}, self.nodes[0].importmempool(mempooldat0, {"apply_fee_delta_priority": True, "apply_unbroadcast_set": True})) assert_equal(2, self.nodes[0].getmempoolinfo()["unbroadcastcount"]) fees = self.nodes[0].getmempoolentry(txid=last_txid)["fees"] assert_equal(fees["base"] + Decimal("0.00001000"), fees["modified"]) self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") self.stop_nodes() self.start_node(0) assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 7) self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) result0 = self.nodes[0].savemempool() assert os.path.isfile(mempooldat0) assert_equal(result0['filename'], mempooldat0) self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 7 transactions") os.rename(mempooldat0, mempooldat1) self.stop_nodes() self.start_node(1, extra_args=["-persistmempool"]) assert self.nodes[1].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[1].getrawmempool()), 7) self.log.debug("Prevent bitcoind from writing mempool.dat to disk. Verify that `savemempool` fails") # to test the exception we are creating a tmp folder called mempool.dat.new # which is an implementation detail that could change and break this test mempooldotnew1 = mempooldat1 + '.new' os.mkdir(mempooldotnew1) assert_raises_rpc_error(-1, "Unable to dump mempool to disk", self.nodes[1].savemempool) os.rmdir(mempooldotnew1) self.test_importmempool_union() self.test_persist_unbroadcast() def test_persist_unbroadcast(self): node0 = self.nodes[0] self.start_node(0) self.start_node(2) # clear out mempool self.generate(node0, 1, sync_fun=self.no_op) # ensure node0 doesn't have any connections # make a transaction that will remain in the unbroadcast set assert_equal(len(node0.getpeerinfo()), 0) assert_equal(len(node0.p2ps), 0) self.mini_wallet.send_self_transfer(from_node=node0) # shutdown, then startup with wallet disabled self.restart_node(0, extra_args=["-disablewallet"]) # check that txn gets broadcast due to unbroadcast logic conn = node0.add_p2p_connection(P2PTxInvStore()) node0.mockscheduler(16 * 60) # 15 min + 1 for buffer self.wait_until(lambda: len(conn.get_invs()) == 1) def test_importmempool_union(self): self.log.debug("Submit different transactions to node0 and node1's mempools") self.start_node(0) self.start_node(2) tx_node0 = self.mini_wallet.send_self_transfer(from_node=self.nodes[0]) tx_node1 = self.mini_wallet.send_self_transfer(from_node=self.nodes[1]) tx_node01 = self.mini_wallet.create_self_transfer() tx_node01_secret = self.mini_wallet.create_self_transfer() self.nodes[0].prioritisetransaction(tx_node01["txid"], 0, COIN) self.nodes[0].prioritisetransaction(tx_node01_secret["txid"], 0, 2 * COIN) self.nodes[1].prioritisetransaction(tx_node01_secret["txid"], 0, 3 * COIN) self.nodes[0].sendrawtransaction(tx_node01["hex"]) self.nodes[1].sendrawtransaction(tx_node01["hex"]) assert tx_node0["txid"] in self.nodes[0].getrawmempool() assert not tx_node0["txid"] in self.nodes[1].getrawmempool() assert not tx_node1["txid"] in self.nodes[0].getrawmempool() assert tx_node1["txid"] in self.nodes[1].getrawmempool() assert tx_node01["txid"] in self.nodes[0].getrawmempool() assert tx_node01["txid"] in self.nodes[1].getrawmempool() assert not tx_node01_secret["txid"] in self.nodes[0].getrawmempool() assert not tx_node01_secret["txid"] in self.nodes[1].getrawmempool() self.log.debug("Check that importmempool can add txns without replacing the entire mempool") mempooldat0 = str(self.nodes[0].chain_path / "mempool.dat") result0 = self.nodes[0].savemempool() assert_equal(mempooldat0, result0["filename"]) assert_equal({}, self.nodes[1].importmempool(mempooldat0, {"apply_fee_delta_priority": True})) # All transactions should be in node1's mempool now. assert tx_node0["txid"] in self.nodes[1].getrawmempool() assert tx_node1["txid"] in self.nodes[1].getrawmempool() assert not tx_node1["txid"] in self.nodes[0].getrawmempool() # For transactions that already existed, priority should be changed entry_node01 = self.nodes[1].getmempoolentry(tx_node01["txid"]) assert_equal(entry_node01["fees"]["base"] + 1, entry_node01["fees"]["modified"]) # Deltas for not-yet-submitted transactions should be applied as well (prioritisation is stackable). self.nodes[1].sendrawtransaction(tx_node01_secret["hex"]) entry_node01_secret = self.nodes[1].getmempoolentry(tx_node01_secret["txid"]) assert_equal(entry_node01_secret["fees"]["base"] + 5, entry_node01_secret["fees"]["modified"]) self.stop_nodes() if __name__ == "__main__": MempoolPersistTest(__file__).main()