diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index e1032ba6bd6..77358f93d95 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -41,6 +41,7 @@ if [ "${RUN_TIDY}" = "true" ]; then CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ " src/compat"\ " src/init"\ + " src/kernel/mempool_persist.cpp"\ " src/policy/feerate.cpp"\ " src/policy/packages.cpp"\ " src/policy/settings.cpp"\ diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp index ce7786f58c2..919ffab102d 100644 --- a/contrib/devtools/iwyu/bitcoin.core.imp +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -3,4 +3,5 @@ { include: [ "", private, "", public ] }, { include: [ "", private, "", public ] }, { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, ] diff --git a/src/Makefile.am b/src/Makefile.am index d293775fc33..02ec5c098eb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -177,6 +177,7 @@ BITCOIN_CORE_H = \ kernel/context.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ + kernel/mempool_persist.h \ key.h \ key_io.h \ logging.h \ @@ -198,6 +199,7 @@ BITCOIN_CORE_H = \ node/chainstate.h \ node/coin.h \ node/context.h \ + node/mempool_persist_args.h \ node/miner.h \ node/minisketchwrapper.h \ node/psbt.h \ @@ -366,6 +368,7 @@ libbitcoin_node_a_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ mapport.cpp \ mempool_args.cpp \ net.cpp \ @@ -379,6 +382,7 @@ libbitcoin_node_a_SOURCES = \ node/context.cpp \ node/eviction.cpp \ node/interfaces.cpp \ + node/mempool_persist_args.cpp \ node/miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ @@ -881,6 +885,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 8922dda3ad4..b43816636f9 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ + test/fuzz/mempool_utils.h \ test/fuzz/util.h libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) diff --git a/src/fs.h b/src/fs.h index cc55793b951..e8b34319bbe 100644 --- a/src/fs.h +++ b/src/fs.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -199,6 +200,7 @@ bool create_directories(const std::filesystem::path& p, std::error_code& ec) = d /** Bridge operations to C stdio */ namespace fsbridge { + using FopenFn = std::function; FILE *fopen(const fs::path& p, const char *mode); /** diff --git a/src/init.cpp b/src/init.cpp index 267070a4c96..97c823fe0c1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -41,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -102,14 +104,19 @@ #include #endif +using kernel::DumpMempool; + using node::CacheSizes; using node::CalculateCacheSizes; using node::ChainstateLoadVerifyError; using node::ChainstateLoadingError; using node::CleanupBlockRevFiles; +using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPAFTERBLOCKIMPORT; using node::LoadChainstate; +using node::MempoolPath; +using node::ShouldPersistMempool; using node::NodeContext; using node::ThreadImport; using node::VerifyLoadedChainstate; @@ -245,8 +252,8 @@ void Shutdown(NodeContext& node) node.addrman.reset(); node.netgroupman.reset(); - if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(*node.mempool); + if (node.mempool && node.mempool->GetLoadTried() && ShouldPersistMempool(*node.args)) { + DumpMempool(*node.mempool, MempoolPath(*node.args)); } // Drop transactions we were still watching, and record fee estimations. @@ -1669,7 +1676,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args); + ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp new file mode 100644 index 00000000000..1a1cf2bbdc1 --- /dev/null +++ b/src/kernel/mempool_persist.cpp @@ -0,0 +1,189 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using fsbridge::FopenFn; + +namespace kernel { + +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, CChainState& active_chainstate, FopenFn mockable_fopen_function) +{ + if (load_path.empty()) return false; + + FILE* filestr{mockable_fopen_function(load_path, "rb")}; + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t expired = 0; + int64_t failed = 0; + int64_t already_there = 0; + int64_t unbroadcast = 0; + auto now = NodeClock::now(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + while (num) { + --num; + CTransactionRef tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); + } + if (nTime > TicksSinceEpoch(now - pool.m_expiry)) { + LOCK(cs_main); + const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); + if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { + ++count; + } else { + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { + ++already_there; + } else { + ++failed; + } + } + } else { + ++expired; + } + if (ShutdownRequested()) + return false; + } + std::map mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } + + std::set unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + return true; +} + +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit) +{ + auto start = SteadyClock::now(); + + std::map mapDeltas; + std::vector vinfo; + std::set unbroadcast_txids; + + static Mutex dump_mutex; + LOCK(dump_mutex); + + { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { + mapDeltas[i.first] = i.second; + } + vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); + } + + auto mid = SteadyClock::now(); + + try { + FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; + if (!filestr) { + return false; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << int64_t{count_seconds(i.m_time)}; + file << int64_t{i.nFeeDelta}; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + + if (!skip_file_commit && !FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); + file.fclose(); + if (!RenameOver(dump_path + ".new", dump_path)) { + throw std::runtime_error("Rename failed"); + } + auto last = SteadyClock::now(); + + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", + Ticks(mid - start), + Ticks(last - mid)); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; + } + return true; +} + +} // namespace kernel diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h new file mode 100644 index 00000000000..9a15ec6dcaf --- /dev/null +++ b/src/kernel/mempool_persist.h @@ -0,0 +1,28 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H + +#include + +class CChainState; +class CTxMemPool; + +namespace kernel { + +/** Dump the mempool to disk. */ +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, + bool skip_file_commit = false); + +/** Load the mempool from disk. */ +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, + CChainState& active_chainstate, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); + +} // namespace kernel + + +#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index cadafcaa8db..103f4f0d7f8 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -823,7 +823,7 @@ struct CImportingNow { } }; -void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args) +void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -893,6 +893,6 @@ void ThreadImport(ChainstateManager& chainman, std::vector vImportFile return; } } // End scope of CImportingNow - chainman.ActiveChainstate().LoadMempool(args); + chainman.ActiveChainstate().LoadMempool(mempool_path); } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index e017f3f4279..9b76371aaec 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -211,7 +211,7 @@ bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, c bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); -void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args); +void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/mempool_persist_args.cpp b/src/node/mempool_persist_args.cpp new file mode 100644 index 00000000000..4e775869c64 --- /dev/null +++ b/src/node/mempool_persist_args.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include + +namespace node { + +bool ShouldPersistMempool(const ArgsManager& argsman) +{ + return argsman.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL); +} + +fs::path MempoolPath(const ArgsManager& argsman) +{ + return argsman.GetDataDirNet() / "mempool.dat"; +} + +} // namespace node diff --git a/src/node/mempool_persist_args.h b/src/node/mempool_persist_args.h new file mode 100644 index 00000000000..f719ec62ab2 --- /dev/null +++ b/src/node/mempool_persist_args.h @@ -0,0 +1,25 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H + +#include + +class ArgsManager; + +namespace node { + +/** + * Default for -persistmempool, indicating whether the node should attempt to + * automatically load the mempool on start and save to disk on shutdown + */ +static constexpr bool DEFAULT_PERSIST_MEMPOOL{true}; + +bool ShouldPersistMempool(const ArgsManager& argsman); +fs::path MempoolPath(const ArgsManager& argsman); + +} // namespace node + +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 84d43e78188..3b53ec82e4b 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,9 +5,12 @@ #include +#include + #include #include #include +#include #include #include #include @@ -18,7 +21,11 @@ #include #include +using kernel::DumpMempool; + using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::MempoolPath; +using node::ShouldPersistMempool; using node::NodeContext; static RPCHelpMan sendrawtransaction() @@ -653,7 +660,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) // Make sure this call is atomic in the pool. LOCK(pool.cs); UniValue ret(UniValue::VOBJ); - ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("loaded", pool.GetLoadTried()); ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); @@ -717,16 +724,18 @@ static RPCHelpMan savemempool() const ArgsManager& args{EnsureAnyArgsman(request.context)}; const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - if (!mempool.IsLoaded()) { + if (!mempool.GetLoadTried()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool(mempool)) { + const fs::path& dump_path = MempoolPath(args); + + if (!DumpMempool(mempool, dump_path)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); + ret.pushKV("filename", dump_path.u8string()); return ret; }, diff --git a/src/test/fuzz/mempool_utils.h b/src/test/fuzz/mempool_utils.h new file mode 100644 index 00000000000..bfe12e30ba0 --- /dev/null +++ b/src/test/fuzz/mempool_utils.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H +#define BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H + +#include + +class DummyChainState final : public CChainState +{ +public: + void SetMempool(CTxMemPool* mempool) + { + m_mempool = mempool; + } +}; + +#endif // BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 2d88ee295bb..63fbf0516a2 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -34,15 +35,6 @@ struct MockedTxPool : public CTxMemPool { } }; -class DummyChainState final : public CChainState -{ -public: - void SetMempool(CTxMemPool* mempool) - { - m_mempool = mempool; - } -}; - void initialize_tx_pool() { static const auto testing_setup = MakeNoLogFileContext(); diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 9532610f8da..90c1a71d9f5 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,10 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include + #include #include +#include #include #include +#include #include #include #include @@ -15,6 +19,10 @@ #include #include +using kernel::DumpMempool; + +using node::MempoolPath; + namespace { const TestingSetup* g_setup; } // namespace @@ -33,9 +41,12 @@ FUZZ_TARGET_INIT(validation_load_mempool, initialize_validation_load_mempool) CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + auto& chainstate{static_cast(g_setup->m_node.chainman->ActiveChainstate())}; + chainstate.SetMempool(&pool); + auto fuzzed_fopen = [&](const fs::path&, const char*) { return fuzzed_file_provider.open(); }; - (void)LoadMempool(pool, g_setup->m_node.chainman->ActiveChainstate(), fuzzed_fopen); - (void)DumpMempool(pool, fuzzed_fopen, true); + (void)chainstate.LoadMempool(MempoolPath(g_setup->m_args), fuzzed_fopen); + (void)DumpMempool(pool, MempoolPath(g_setup->m_args), fuzzed_fopen, true); } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index aeaa10034eb..7eff6bdbe32 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1210,14 +1210,14 @@ void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, } } -bool CTxMemPool::IsLoaded() const +bool CTxMemPool::GetLoadTried() const { LOCK(cs); - return m_is_loaded; + return m_load_tried; } -void CTxMemPool::SetIsLoaded(bool loaded) +void CTxMemPool::SetLoadTried(bool load_tried) { LOCK(cs); - m_is_loaded = loaded; + m_load_tried = load_tried; } diff --git a/src/txmempool.h b/src/txmempool.h index 6e37f59f2e9..d7d308038c2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -451,7 +451,7 @@ protected: void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool m_is_loaded GUARDED_BY(cs){false}; + bool m_load_tried GUARDED_BY(cs){false}; CFeeRate GetMinFee(size_t sizelimit) const; @@ -728,11 +728,17 @@ public: */ void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; - /** @returns true if the mempool is fully loaded */ - bool IsLoaded() const; + /** + * @returns true if we've made an attempt to load the mempool regardless of + * whether the attempt was successful or not + */ + bool GetLoadTried() const; - /** Sets the current loaded state */ - void SetIsLoaded(bool loaded); + /** + * Set whether or not we've made an attempt to load the mempool (regardless + * of whether the attempt was successful or not) + */ + void SetLoadTried(bool load_tried); unsigned long size() const { diff --git a/src/util/time.h b/src/util/time.h index 9df69a953c6..fc49f23ce31 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -24,6 +24,7 @@ struct NodeClock : public std::chrono::system_clock { }; using NodeSeconds = std::chrono::time_point; +using SteadyClock = std::chrono::steady_clock; using SteadySeconds = std::chrono::time_point; using SteadyMilliseconds = std::chrono::time_point; using SteadyMicroseconds = std::chrono::time_point; diff --git a/src/validation.cpp b/src/validation.cpp index 4c694a2c21c..6840753cd4a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,6 +5,9 @@ #include +#include +#include + #include #include #include @@ -17,8 +20,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -47,12 +50,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -61,7 +66,9 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; +using kernel::LoadMempool; +using fsbridge::FopenFn; using node::BLOCKFILE_CHUNK_SIZE; using node::BlockManager; using node::BlockMap; @@ -3861,13 +3868,11 @@ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeigh } } -void CChainState::LoadMempool(const ArgsManager& args) +void CChainState::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) { if (!m_mempool) return; - if (args.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(*m_mempool, *this); - } - m_mempool->SetIsLoaded(!ShutdownRequested()); + ::LoadMempool(*m_mempool, load_path, *this, mockable_fopen_function); + m_mempool->SetLoadTried(!ShutdownRequested()); } bool CChainState::LoadChainTip() @@ -4638,153 +4643,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -static const uint64_t MEMPOOL_DUMP_VERSION = 1; - -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) -{ - int64_t nExpiryTimeout = std::chrono::seconds{pool.m_expiry}.count(); - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); - return false; - } - - int64_t count = 0; - int64_t expired = 0; - int64_t failed = 0; - int64_t already_there = 0; - int64_t unbroadcast = 0; - int64_t nNow = GetTime(); - - try { - uint64_t version; - file >> version; - if (version != MEMPOOL_DUMP_VERSION) { - return false; - } - uint64_t num; - file >> num; - while (num) { - --num; - CTransactionRef tx; - int64_t nTime; - int64_t nFeeDelta; - file >> tx; - file >> nTime; - file >> nFeeDelta; - - CAmount amountdelta = nFeeDelta; - if (amountdelta) { - pool.PrioritiseTransaction(tx->GetHash(), amountdelta); - } - if (nTime > nNow - nExpiryTimeout) { - LOCK(cs_main); - const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); - if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { - ++count; - } else { - // mempool may contain the transaction already, e.g. from - // wallet(s) having loaded it while we were processing - // mempool transactions; consider these as valid, instead of - // failed, but mark them as 'already there' - if (pool.exists(GenTxid::Txid(tx->GetHash()))) { - ++already_there; - } else { - ++failed; - } - } - } else { - ++expired; - } - if (ShutdownRequested()) - return false; - } - std::map mapDeltas; - file >> mapDeltas; - - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); - } - - std::set unbroadcast_txids; - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); - } - } catch (const std::exception& e) { - LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); - return false; - } - - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); - return true; -} - -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function, bool skip_file_commit) -{ - int64_t start = GetTimeMicros(); - - std::map mapDeltas; - std::vector vinfo; - std::set unbroadcast_txids; - - static Mutex dump_mutex; - LOCK(dump_mutex); - - { - LOCK(pool.cs); - for (const auto &i : pool.mapDeltas) { - mapDeltas[i.first] = i.second; - } - vinfo = pool.infoAll(); - unbroadcast_txids = pool.GetUnbroadcastTxs(); - } - - int64_t mid = GetTimeMicros(); - - try { - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat.new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - - uint64_t version = MEMPOOL_DUMP_VERSION; - file << version; - - file << (uint64_t)vinfo.size(); - for (const auto& i : vinfo) { - file << *(i.tx); - file << int64_t{count_seconds(i.m_time)}; - file << int64_t{i.nFeeDelta}; - mapDeltas.erase(i.tx->GetHash()); - } - - file << mapDeltas; - - LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); - file << unbroadcast_txids; - - if (!skip_file_commit && !FileCommit(file.Get())) - throw std::runtime_error("FileCommit failed"); - file.fclose(); - if (!RenameOver(gArgs.GetDataDirNet() / "mempool.dat.new", gArgs.GetDataDirNet() / "mempool.dat")) { - throw std::runtime_error("Rename failed"); - } - int64_t last = GetTimeMicros(); - LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); - } catch (const std::exception& e) { - LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); - return false; - } - return true; -} - //! Guess how far we are in the verification process at the given block index //! require cs_main if pindex has not been validated yet (because nChainTx might be unset) double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { diff --git a/src/validation.h b/src/validation.h index aa9fcf9f2da..a21d9e3a28d 100644 --- a/src/validation.h +++ b/src/validation.h @@ -68,8 +68,6 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; -/** Default for -persistmempool */ -static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ @@ -679,7 +677,7 @@ public: void CheckBlockIndex(); /** Load the persisted mempool from disk */ - void LoadMempool(const ArgsManager& args); + void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -1014,14 +1012,6 @@ bool DeploymentEnabled(const ChainstateManager& chainman, DEP dep) return DeploymentEnabled(chainman.GetConsensus(), dep); } -using FopenFn = std::function; - -/** Dump the mempool to disk. */ -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbridge::fopen, bool skip_file_commit = false); - -/** Load the mempool from disk. */ -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 58f4c913434..b6fa7fbd919 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -141,6 +141,16 @@ class MempoolPersistTest(BitcoinTestFramework): self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, wallet_watch.getbalance()) + mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') + mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') + + self.log.debug("Force -persistmempool=0 node1 to savemempool to disk via RPC") + assert not os.path.exists(mempooldat1) + result1 = self.nodes[1].savemempool() + assert os.path.isfile(mempooldat1) + assert_equal(result1['filename'], mempooldat1) + os.remove(mempooldat1) + self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) @@ -153,8 +163,6 @@ class MempoolPersistTest(BitcoinTestFramework): assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 7) - mempooldat0 = os.path.join(self.nodes[0].datadir, self.chain, 'mempool.dat') - mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) result0 = self.nodes[0].savemempool() diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py index 5d157eb4b1b..f44ed8f7c7e 100755 --- a/test/lint/lint-circular-dependencies.py +++ b/test/lint/lint-circular-dependencies.py @@ -22,6 +22,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES = ( "wallet/fees -> wallet/wallet -> wallet/fees", "wallet/wallet -> wallet/walletdb -> wallet/wallet", "kernel/coinstats -> validation -> kernel/coinstats", + "kernel/mempool_persist -> validation -> kernel/mempool_persist", ) CODE_DIR = "src"