mirror of
https://github.com/bitcoin/bitcoin.git
synced 2025-02-22 23:07:59 +01:00
Merge bitcoin/bitcoin#31483: kernel: Move kernel-related cache constants to kernel cache
2a92702bafinit: Use size_t consistently for cache sizes (TheCharlatan)65cde3621dkernel: Move default cache constants to caches (TheCharlatan)8826cae285kernel: Move non-kernel db cache size constants (TheCharlatan)e758b26b85kernel: Move kernel-specific cache size options to kernel (TheCharlatan)d5e2c4a409fuzz: Add fuzz test for checked and saturating add and left shift (TheCharlatan)c03a2795a8util: Add integer left shift helpers (TheCharlatan)8bd5f8a38c[refactor] init: Simplify coinsdb cache calculation (TheCharlatan)5db7d4d3d2doc: Correct docstring describing max block tree db cache (TheCharlatan) Pull request description: Carrying non-kernel related fields in the cache sizes for the indexes is confusing for kernel library users. The cache sizes are set currently with magic numbers in bitcoin-chainstate. The comments for the cache size calculations are not completely clear. The constants for the cache sizes are also currently in `txdb.h`, which is not an ideal place for holding all cache size related constants. Solve these things by moving the kernel-specific cache size fields to their own struct and moving the constants to either the node or the kernel cache sizes. This slightly changes the way the cache is allocated if (and only if) the txindex and/or blockfilterindex is used. Since they are now given precedence over the block tree db cache, this results in a bit less cache being allocated to the block tree db, coinsdb and coins caches. The effect is negligible though, i.e. cache sizes with default dbcache reported through the logs are: master: ``` Cache configuration: * Using 2.0 MiB for block index database * Using 56.0 MiB for transaction index database * Using 49.0 MiB for basic block filter index database * Using 8.0 MiB for chain state database * Using 335.0 MiB for in-memory UTXO set (plus up to 286.1 MiB of unused mempool space) ``` this PR: ``` Cache configuration: * Using 2.0 MiB for block index database * Using 56.2 MiB for transaction index database * Using 49.2 MiB for basic block filter index database * Using 8.0 MiB for chain state database * Using 334.5 MiB for in-memory UTXO set (plus up to 286.1 MiB of unused mempool space) ``` --- This PR is part of the [libbitcoinkernel project](https://github.com/bitcoin/bitcoin/issues/27587). ACKs for top commit: stickies-v: re-ACK2a92702bafryanofsky: Code review ACK2a92702baf. Changes since last review are fixing size options to use size_t instead of int64_t again, simplifying CheckedLeftShift more, and making other minor suggested cleanups hodlinator: re-ACK2a92702bafTree-SHA512: 98376eaa0660b1b8c096a5ce1f3e7c8c30e7cd6644de36856c2d3e573108cfc9473c93ebb3952b7881047b5ae6c85c5b096e6726f30f35be58b98eca07c8c785
This commit is contained in:
merge-script
commit
df8bf65745
18 changed files with 322 additions and 76 deletions
|
|
@ -19,9 +19,9 @@
|
|||
|
||||
#include <consensus/validation.h>
|
||||
#include <core_io.h>
|
||||
#include <kernel/caches.h>
|
||||
#include <logging.h>
|
||||
#include <node/blockstorage.h>
|
||||
#include <node/caches.h>
|
||||
#include <node/chainstate.h>
|
||||
#include <random.h>
|
||||
#include <script/sigcache.h>
|
||||
|
|
@ -123,10 +123,7 @@ int main(int argc, char* argv[])
|
|||
util::SignalInterrupt interrupt;
|
||||
ChainstateManager chainman{interrupt, chainman_opts, blockman_opts};
|
||||
|
||||
node::CacheSizes cache_sizes;
|
||||
cache_sizes.block_tree_db = 2 << 20;
|
||||
cache_sizes.coins_db = 2 << 22;
|
||||
cache_sizes.coins = (450 << 20) - (2 << 20) - (2 << 22);
|
||||
kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
|
||||
node::ChainstateLoadOptions options;
|
||||
auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options);
|
||||
if (status != node::ChainstateLoadStatus::SUCCESS) {
|
||||
|
|
|
|||
28
src/init.cpp
28
src/init.cpp
|
|
@ -32,6 +32,7 @@
|
|||
#include <interfaces/ipc.h>
|
||||
#include <interfaces/mining.h>
|
||||
#include <interfaces/node.h>
|
||||
#include <kernel/caches.h>
|
||||
#include <kernel/context.h>
|
||||
#include <key.h>
|
||||
#include <logging.h>
|
||||
|
|
@ -121,7 +122,6 @@ using common::ResolveErrMsg;
|
|||
|
||||
using node::ApplyArgsManOptions;
|
||||
using node::BlockManager;
|
||||
using node::CacheSizes;
|
||||
using node::CalculateCacheSizes;
|
||||
using node::ChainstateLoadResult;
|
||||
using node::ChainstateLoadStatus;
|
||||
|
|
@ -487,7 +487,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
|
|||
argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", nMinDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", MIN_DB_CACHE >> 20, DEFAULT_DB_CACHE >> 20), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
|
|
@ -1177,7 +1177,7 @@ static ChainstateLoadResult InitAndLoadChainstate(
|
|||
NodeContext& node,
|
||||
bool do_reindex,
|
||||
const bool do_reindex_chainstate,
|
||||
CacheSizes& cache_sizes,
|
||||
const kernel::CacheSizes& cache_sizes,
|
||||
const ArgsManager& args)
|
||||
{
|
||||
const CChainParams& chainparams = Params();
|
||||
|
|
@ -1602,18 +1602,18 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
|
|||
ReadNotificationArgs(args, kernel_notifications);
|
||||
|
||||
// cache size calculations
|
||||
CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size());
|
||||
const auto [index_cache_sizes, kernel_cache_sizes] = CalculateCacheSizes(args, g_enabled_filter_types.size());
|
||||
|
||||
LogPrintf("Cache configuration:\n");
|
||||
LogPrintf("* Using %.1f MiB for block index database\n", cache_sizes.block_tree_db * (1.0 / 1024 / 1024));
|
||||
LogInfo("Cache configuration:");
|
||||
LogInfo("* Using %.1f MiB for block index database", kernel_cache_sizes.block_tree_db * (1.0 / 1024 / 1024));
|
||||
if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
|
||||
LogPrintf("* Using %.1f MiB for transaction index database\n", cache_sizes.tx_index * (1.0 / 1024 / 1024));
|
||||
LogInfo("* Using %.1f MiB for transaction index database", index_cache_sizes.tx_index * (1.0 / 1024 / 1024));
|
||||
}
|
||||
for (BlockFilterType filter_type : g_enabled_filter_types) {
|
||||
LogPrintf("* Using %.1f MiB for %s block filter index database\n",
|
||||
cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
|
||||
LogInfo("* Using %.1f MiB for %s block filter index database",
|
||||
index_cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
|
||||
}
|
||||
LogPrintf("* Using %.1f MiB for chain state database\n", cache_sizes.coins_db * (1.0 / 1024 / 1024));
|
||||
LogInfo("* Using %.1f MiB for chain state database", kernel_cache_sizes.coins_db * (1.0 / 1024 / 1024));
|
||||
|
||||
assert(!node.mempool);
|
||||
assert(!node.chainman);
|
||||
|
|
@ -1626,7 +1626,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
|
|||
node,
|
||||
do_reindex,
|
||||
do_reindex_chainstate,
|
||||
cache_sizes,
|
||||
kernel_cache_sizes,
|
||||
args);
|
||||
if (status == ChainstateLoadStatus::FAILURE && !do_reindex && !ShutdownRequested(node)) {
|
||||
// suggest a reindex
|
||||
|
|
@ -1645,7 +1645,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
|
|||
node,
|
||||
do_reindex,
|
||||
do_reindex_chainstate,
|
||||
cache_sizes,
|
||||
kernel_cache_sizes,
|
||||
args);
|
||||
}
|
||||
if (status != ChainstateLoadStatus::SUCCESS && status != ChainstateLoadStatus::INTERRUPTED) {
|
||||
|
|
@ -1671,12 +1671,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
|
|||
// ********************************************************* Step 8: start indexers
|
||||
|
||||
if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
|
||||
g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, do_reindex);
|
||||
g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), index_cache_sizes.tx_index, false, do_reindex);
|
||||
node.indexes.emplace_back(g_txindex.get());
|
||||
}
|
||||
|
||||
for (const auto& filter_type : g_enabled_filter_types) {
|
||||
InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, do_reindex);
|
||||
InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, index_cache_sizes.filter_index, false, do_reindex);
|
||||
node.indexes.emplace_back(GetBlockFilterIndex(filter_type));
|
||||
}
|
||||
|
||||
|
|
|
|||
36
src/kernel/caches.h
Normal file
36
src/kernel/caches.h
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// Copyright (c) 2024-present The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#ifndef BITCOIN_KERNEL_CACHES_H
|
||||
#define BITCOIN_KERNEL_CACHES_H
|
||||
|
||||
#include <util/byte_units.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
//! Suggested default amount of cache reserved for the kernel (bytes)
|
||||
static constexpr size_t DEFAULT_KERNEL_CACHE{450_MiB};
|
||||
//! Max memory allocated to block tree DB specific cache (bytes)
|
||||
static constexpr size_t MAX_BLOCK_DB_CACHE{2_MiB};
|
||||
//! Max memory allocated to coin DB specific cache (bytes)
|
||||
static constexpr size_t MAX_COINS_DB_CACHE{8_MiB};
|
||||
|
||||
namespace kernel {
|
||||
struct CacheSizes {
|
||||
size_t block_tree_db;
|
||||
size_t coins_db;
|
||||
size_t coins;
|
||||
|
||||
CacheSizes(size_t total_cache)
|
||||
{
|
||||
block_tree_db = std::min(total_cache / 8, MAX_BLOCK_DB_CACHE);
|
||||
total_cache -= block_tree_db;
|
||||
coins_db = std::min(total_cache / 2, MAX_COINS_DB_CACHE);
|
||||
total_cache -= coins_db;
|
||||
coins = total_cache; // the rest goes to the coins cache
|
||||
}
|
||||
};
|
||||
} // namespace kernel
|
||||
|
||||
#endif // BITCOIN_KERNEL_CACHES_H
|
||||
|
|
@ -6,28 +6,39 @@
|
|||
|
||||
#include <common/args.h>
|
||||
#include <index/txindex.h>
|
||||
#include <txdb.h>
|
||||
#include <kernel/caches.h>
|
||||
#include <logging.h>
|
||||
#include <util/byte_units.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
|
||||
// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
|
||||
// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
|
||||
//! Max memory allocated to tx index DB specific cache in bytes.
|
||||
static constexpr size_t MAX_TX_INDEX_CACHE{1024_MiB};
|
||||
//! Max memory allocated to all block filter index caches combined in bytes.
|
||||
static constexpr size_t MAX_FILTER_INDEX_CACHE{1024_MiB};
|
||||
|
||||
namespace node {
|
||||
CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
|
||||
{
|
||||
int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20);
|
||||
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
|
||||
CacheSizes sizes;
|
||||
sizes.block_tree_db = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
|
||||
nTotalCache -= sizes.block_tree_db;
|
||||
sizes.tx_index = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
|
||||
nTotalCache -= sizes.tx_index;
|
||||
sizes.filter_index = 0;
|
||||
if (n_indexes > 0) {
|
||||
int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
|
||||
sizes.filter_index = max_cache / n_indexes;
|
||||
nTotalCache -= sizes.filter_index * n_indexes;
|
||||
// Convert -dbcache from MiB units to bytes. The total cache is floored by MIN_DB_CACHE and capped by max size_t value.
|
||||
size_t total_cache{DEFAULT_DB_CACHE};
|
||||
if (std::optional<int64_t> db_cache = args.GetIntArg("-dbcache")) {
|
||||
if (*db_cache < 0) db_cache = 0;
|
||||
uint64_t db_cache_bytes = SaturatingLeftShift<uint64_t>(*db_cache, 20);
|
||||
total_cache = std::max<size_t>(MIN_DB_CACHE, std::min<uint64_t>(db_cache_bytes, std::numeric_limits<size_t>::max()));
|
||||
}
|
||||
sizes.coins_db = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
|
||||
sizes.coins_db = std::min(sizes.coins_db, nMaxCoinsDBCache << 20); // cap total coins db cache
|
||||
nTotalCache -= sizes.coins_db;
|
||||
sizes.coins = nTotalCache; // the rest goes to in-memory cache
|
||||
return sizes;
|
||||
|
||||
IndexCacheSizes index_sizes;
|
||||
index_sizes.tx_index = std::min(total_cache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? MAX_TX_INDEX_CACHE : 0);
|
||||
total_cache -= index_sizes.tx_index;
|
||||
if (n_indexes > 0) {
|
||||
size_t max_cache = std::min(total_cache / 8, MAX_FILTER_INDEX_CACHE);
|
||||
index_sizes.filter_index = max_cache / n_indexes;
|
||||
total_cache -= index_sizes.filter_index * n_indexes;
|
||||
}
|
||||
return {index_sizes, kernel::CacheSizes{total_cache}};
|
||||
}
|
||||
} // namespace node
|
||||
|
|
|
|||
|
|
@ -5,18 +5,26 @@
|
|||
#ifndef BITCOIN_NODE_CACHES_H
|
||||
#define BITCOIN_NODE_CACHES_H
|
||||
|
||||
#include <kernel/caches.h>
|
||||
#include <util/byte_units.h>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
class ArgsManager;
|
||||
|
||||
//! min. -dbcache (bytes)
|
||||
static constexpr size_t MIN_DB_CACHE{4_MiB};
|
||||
//! -dbcache default (bytes)
|
||||
static constexpr size_t DEFAULT_DB_CACHE{DEFAULT_KERNEL_CACHE};
|
||||
|
||||
namespace node {
|
||||
struct IndexCacheSizes {
|
||||
size_t tx_index{0};
|
||||
size_t filter_index{0};
|
||||
};
|
||||
struct CacheSizes {
|
||||
int64_t block_tree_db;
|
||||
int64_t coins_db;
|
||||
int64_t coins;
|
||||
int64_t tx_index;
|
||||
int64_t filter_index;
|
||||
IndexCacheSizes index;
|
||||
kernel::CacheSizes kernel;
|
||||
};
|
||||
CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
|
||||
} // namespace node
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@
|
|||
#include <chain.h>
|
||||
#include <coins.h>
|
||||
#include <consensus/params.h>
|
||||
#include <kernel/caches.h>
|
||||
#include <logging.h>
|
||||
#include <node/blockstorage.h>
|
||||
#include <node/caches.h>
|
||||
#include <sync.h>
|
||||
#include <threadsafety.h>
|
||||
#include <tinyformat.h>
|
||||
|
|
@ -29,6 +29,8 @@
|
|||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
using kernel::CacheSizes;
|
||||
|
||||
namespace node {
|
||||
// Complete initialization of chainstates after the initial call has been made
|
||||
// to ChainstateManager::InitializeChainstate().
|
||||
|
|
@ -44,7 +46,7 @@ static ChainstateLoadResult CompleteChainstateInitialization(
|
|||
try {
|
||||
pblocktree = std::make_unique<BlockTreeDB>(DBParams{
|
||||
.path = chainman.m_options.datadir / "blocks" / "index",
|
||||
.cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db),
|
||||
.cache_bytes = cache_sizes.block_tree_db,
|
||||
.memory_only = options.block_tree_db_in_memory,
|
||||
.wipe_data = options.wipe_block_tree_db,
|
||||
.options = chainman.m_options.block_tree_db});
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@
|
|||
|
||||
class CTxMemPool;
|
||||
|
||||
namespace node {
|
||||
|
||||
namespace kernel {
|
||||
struct CacheSizes;
|
||||
} // namespace kernel
|
||||
|
||||
namespace node {
|
||||
|
||||
struct ChainstateLoadOptions {
|
||||
CTxMemPool* mempool{nullptr};
|
||||
|
|
@ -69,7 +71,7 @@ using ChainstateLoadResult = std::tuple<ChainstateLoadStatus, bilingual_str>;
|
|||
*
|
||||
* LoadChainstate returns a (status code, error string) tuple.
|
||||
*/
|
||||
ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes,
|
||||
ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const kernel::CacheSizes& cache_sizes,
|
||||
const ChainstateLoadOptions& options);
|
||||
ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options);
|
||||
} // namespace node
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@
|
|||
|
||||
#include <common/system.h>
|
||||
#include <interfaces/node.h>
|
||||
#include <node/chainstatemanager_args.h>
|
||||
#include <netbase.h>
|
||||
#include <txdb.h>
|
||||
#include <node/caches.h>
|
||||
#include <node/chainstatemanager_args.h>
|
||||
#include <util/strencodings.h>
|
||||
|
||||
#include <chrono>
|
||||
|
|
@ -95,7 +95,7 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet)
|
|||
ui->verticalLayout->setStretchFactor(ui->tabWidget, 1);
|
||||
|
||||
/* Main elements init */
|
||||
ui->databaseCache->setRange(nMinDbCache, std::numeric_limits<int>::max());
|
||||
ui->databaseCache->setRange(MIN_DB_CACHE >> 20, std::numeric_limits<int>::max());
|
||||
ui->threadsScriptVerif->setMinimum(-GetNumCores());
|
||||
ui->threadsScriptVerif->setMaximum(MAX_SCRIPTCHECK_THREADS);
|
||||
ui->pruneWarning->setVisible(false);
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@
|
|||
#include <mapport.h>
|
||||
#include <net.h>
|
||||
#include <netbase.h>
|
||||
#include <node/caches.h>
|
||||
#include <node/chainstatemanager_args.h>
|
||||
#include <txdb.h> // for -dbcache defaults
|
||||
#include <util/string.h>
|
||||
#include <validation.h> // For DEFAULT_SCRIPTCHECK_THREADS
|
||||
#include <wallet/wallet.h> // For DEFAULT_SPEND_ZEROCONF_CHANGE
|
||||
|
|
@ -470,7 +470,7 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con
|
|||
suffix.empty() ? getOption(option, "-prev") :
|
||||
DEFAULT_PRUNE_TARGET_GB;
|
||||
case DatabaseCache:
|
||||
return qlonglong(SettingToInt(setting(), nDefaultDbCache));
|
||||
return qlonglong(SettingToInt(setting(), DEFAULT_DB_CACHE >> 20));
|
||||
case ThreadsScriptVerif:
|
||||
return qlonglong(SettingToInt(setting(), DEFAULT_SCRIPTCHECK_THREADS));
|
||||
case Listen:
|
||||
|
|
@ -733,7 +733,7 @@ void OptionsModel::checkAndMigrate()
|
|||
// see https://github.com/bitcoin/bitcoin/pull/8273
|
||||
// force people to upgrade to the new value if they are using 100MB
|
||||
if (settingsVersion < 130000 && settings.contains("nDatabaseCache") && settings.value("nDatabaseCache").toLongLong() == 100)
|
||||
settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache);
|
||||
settings.setValue("nDatabaseCache", (qint64)(DEFAULT_DB_CACHE >> 20));
|
||||
|
||||
settings.setValue(strSettingsVersionKey, CLIENT_VERSION);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ add_executable(fuzz
|
|||
netaddress.cpp
|
||||
netbase_dns_lookup.cpp
|
||||
node_eviction.cpp
|
||||
overflow.cpp
|
||||
p2p_handshake.cpp
|
||||
p2p_headers_presync.cpp
|
||||
p2p_transport_serialization.cpp
|
||||
|
|
|
|||
50
src/test/fuzz/overflow.cpp
Normal file
50
src/test/fuzz/overflow.cpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
// Copyright (c) 2025-present The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include <test/fuzz/FuzzedDataProvider.h>
|
||||
#include <test/fuzz/fuzz.h>
|
||||
#include <util/check.h>
|
||||
#include <util/overflow.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
|
||||
namespace {
|
||||
//! Test overflow operations for type T using a wider type, W, to verify results.
|
||||
template <typename T, typename W>
|
||||
void TestOverflow(FuzzedDataProvider& fuzzed_data_provider)
|
||||
{
|
||||
constexpr auto min{std::numeric_limits<T>::min()};
|
||||
constexpr auto max{std::numeric_limits<T>::max()};
|
||||
// Range needs to be at least twice as big to allow two numbers to be added without overflowing.
|
||||
static_assert(min >= std::numeric_limits<W>::min() / 2);
|
||||
static_assert(max <= std::numeric_limits<W>::max() / 2);
|
||||
|
||||
auto widen = [](T value) -> W { return value; };
|
||||
auto clamp = [](W value) -> W { return std::clamp<W>(value, min, max); };
|
||||
auto check = [](W value) -> std::optional<W> { if (value >= min && value <= max) return value; else return std::nullopt; };
|
||||
|
||||
const T i = fuzzed_data_provider.ConsumeIntegral<T>();
|
||||
const T j = fuzzed_data_provider.ConsumeIntegral<T>();
|
||||
const unsigned shift = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, std::numeric_limits<W>::digits - std::numeric_limits<T>::digits);
|
||||
|
||||
Assert(clamp(widen(i) + widen(j)) == SaturatingAdd(i, j));
|
||||
Assert(check(widen(i) + widen(j)) == CheckedAdd(i, j));
|
||||
|
||||
Assert(clamp(widen(i) << shift) == SaturatingLeftShift(i, shift));
|
||||
Assert(check(widen(i) << shift) == CheckedLeftShift(i, shift));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
FUZZ_TARGET(overflow)
|
||||
{
|
||||
FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
|
||||
TestOverflow<int8_t, int64_t>(fuzzed_data_provider);
|
||||
TestOverflow<int16_t, int64_t>(fuzzed_data_provider);
|
||||
TestOverflow<int32_t, int64_t>(fuzzed_data_provider);
|
||||
TestOverflow<uint8_t, uint64_t>(fuzzed_data_provider);
|
||||
TestOverflow<uint16_t, uint64_t>(fuzzed_data_provider);
|
||||
TestOverflow<uint32_t, uint64_t>(fuzzed_data_provider);
|
||||
}
|
||||
|
|
@ -66,7 +66,6 @@ using kernel::BlockTreeDB;
|
|||
using node::ApplyArgsManOptions;
|
||||
using node::BlockAssembler;
|
||||
using node::BlockManager;
|
||||
using node::CalculateCacheSizes;
|
||||
using node::KernelNotifications;
|
||||
using node::LoadChainstate;
|
||||
using node::RegenerateCommitments;
|
||||
|
|
@ -231,8 +230,6 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
|
|||
Assert(error.empty());
|
||||
m_node.warnings = std::make_unique<node::Warnings>();
|
||||
|
||||
m_cache_sizes = CalculateCacheSizes(m_args);
|
||||
|
||||
m_node.notifications = std::make_unique<KernelNotifications>(Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings));
|
||||
|
||||
m_make_chainman = [this, &chainparams, opts] {
|
||||
|
|
@ -258,7 +255,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
|
|||
LOCK(m_node.chainman->GetMutex());
|
||||
m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{
|
||||
.path = m_args.GetDataDirNet() / "blocks" / "index",
|
||||
.cache_bytes = static_cast<size_t>(m_cache_sizes.block_tree_db),
|
||||
.cache_bytes = m_kernel_cache_sizes.block_tree_db,
|
||||
.memory_only = true,
|
||||
});
|
||||
};
|
||||
|
|
@ -294,7 +291,7 @@ void ChainTestingSetup::LoadVerifyActivateChainstate()
|
|||
options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);
|
||||
options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL);
|
||||
options.require_full_verification = m_args.IsArgSet("-checkblocks") || m_args.IsArgSet("-checklevel");
|
||||
auto [status, error] = LoadChainstate(chainman, m_cache_sizes, options);
|
||||
auto [status, error] = LoadChainstate(chainman, m_kernel_cache_sizes, options);
|
||||
assert(status == node::ChainstateLoadStatus::SUCCESS);
|
||||
|
||||
std::tie(status, error) = VerifyLoadedChainstate(chainman, options);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
#define BITCOIN_TEST_UTIL_SETUP_COMMON_H
|
||||
|
||||
#include <common/args.h> // IWYU pragma: export
|
||||
#include <kernel/caches.h>
|
||||
#include <kernel/context.h>
|
||||
#include <key.h>
|
||||
#include <node/caches.h>
|
||||
|
|
@ -103,7 +104,7 @@ struct BasicTestingSetup {
|
|||
* initialization behaviour.
|
||||
*/
|
||||
struct ChainTestingSetup : public BasicTestingSetup {
|
||||
node::CacheSizes m_cache_sizes{};
|
||||
kernel::CacheSizes m_kernel_cache_sizes{node::CalculateCacheSizes(m_args).kernel};
|
||||
bool m_coins_db_in_memory{true};
|
||||
bool m_block_tree_db_in_memory{true};
|
||||
std::function<void()> m_make_chainman{};
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
#include <test/util/setup_common.h>
|
||||
#include <uint256.h>
|
||||
#include <util/bitdeque.h>
|
||||
#include <util/byte_units.h>
|
||||
#include <util/fs.h>
|
||||
#include <util/fs_helpers.h>
|
||||
#include <util/moneystr.h>
|
||||
|
|
@ -1877,4 +1878,100 @@ BOOST_AUTO_TEST_CASE(clearshrink_test)
|
|||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void TestCheckedLeftShift()
|
||||
{
|
||||
constexpr auto MAX{std::numeric_limits<T>::max()};
|
||||
|
||||
// Basic operations
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(0, 1), 0);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(0, 127), 0);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(1, 1), 2);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(2, 2), 8);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(MAX >> 1, 1), MAX - 1);
|
||||
|
||||
// Max left shift
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(1, std::numeric_limits<T>::digits - 1), MAX / 2 + 1);
|
||||
|
||||
// Overflow cases
|
||||
BOOST_CHECK(!CheckedLeftShift<T>((MAX >> 1) + 1, 1));
|
||||
BOOST_CHECK(!CheckedLeftShift<T>(MAX, 1));
|
||||
BOOST_CHECK(!CheckedLeftShift<T>(1, std::numeric_limits<T>::digits));
|
||||
BOOST_CHECK(!CheckedLeftShift<T>(1, std::numeric_limits<T>::digits + 1));
|
||||
|
||||
if constexpr (std::is_signed_v<T>) {
|
||||
constexpr auto MIN{std::numeric_limits<T>::min()};
|
||||
// Negative input
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(-1, 1), -2);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>((MIN >> 2), 1), MIN / 2);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>((MIN >> 1) + 1, 1), MIN + 2);
|
||||
BOOST_CHECK_EQUAL(CheckedLeftShift<T>(MIN >> 1, 1), MIN);
|
||||
// Overflow negative
|
||||
BOOST_CHECK(!CheckedLeftShift<T>((MIN >> 1) - 1, 1));
|
||||
BOOST_CHECK(!CheckedLeftShift<T>(MIN >> 1, 2));
|
||||
BOOST_CHECK(!CheckedLeftShift<T>(-1, 100));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void TestSaturatingLeftShift()
|
||||
{
|
||||
constexpr auto MAX{std::numeric_limits<T>::max()};
|
||||
|
||||
// Basic operations
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(0, 1), 0);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(0, 127), 0);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(1, 1), 2);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(2, 2), 8);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(MAX >> 1, 1), MAX - 1);
|
||||
|
||||
// Max left shift
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(1, std::numeric_limits<T>::digits - 1), MAX / 2 + 1);
|
||||
|
||||
// Saturation cases
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>((MAX >> 1) + 1, 1), MAX);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(MAX, 1), MAX);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(1, std::numeric_limits<T>::digits), MAX);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(1, std::numeric_limits<T>::digits + 1), MAX);
|
||||
|
||||
if constexpr (std::is_signed_v<T>) {
|
||||
constexpr auto MIN{std::numeric_limits<T>::min()};
|
||||
// Negative input
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(-1, 1), -2);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>((MIN >> 2), 1), MIN / 2);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>((MIN >> 1) + 1, 1), MIN + 2);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(MIN >> 1, 1), MIN);
|
||||
// Saturation negative
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>((MIN >> 1) - 1, 1), MIN);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(MIN >> 1, 2), MIN);
|
||||
BOOST_CHECK_EQUAL(SaturatingLeftShift<T>(-1, 100), MIN);
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(checked_left_shift_test)
|
||||
{
|
||||
TestCheckedLeftShift<uint8_t>();
|
||||
TestCheckedLeftShift<int8_t>();
|
||||
TestCheckedLeftShift<size_t>();
|
||||
TestCheckedLeftShift<uint64_t>();
|
||||
TestCheckedLeftShift<int64_t>();
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(saturating_left_shift_test)
|
||||
{
|
||||
TestSaturatingLeftShift<uint8_t>();
|
||||
TestSaturatingLeftShift<int8_t>();
|
||||
TestSaturatingLeftShift<size_t>();
|
||||
TestSaturatingLeftShift<uint64_t>();
|
||||
TestSaturatingLeftShift<int64_t>();
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_CASE(mib_string_literal_test)
|
||||
{
|
||||
BOOST_CHECK_EQUAL(0_MiB, 0);
|
||||
BOOST_CHECK_EQUAL(1_MiB, 1024 * 1024);
|
||||
const auto max_mib{std::numeric_limits<size_t>::max() >> 20};
|
||||
BOOST_CHECK_EXCEPTION(operator""_MiB(static_cast<unsigned long long>(max_mib) + 1), std::overflow_error, HasReason("MiB value too large for size_t byte conversion"));
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END()
|
||||
|
|
|
|||
14
src/txdb.h
14
src/txdb.h
|
|
@ -21,22 +21,8 @@
|
|||
class COutPoint;
|
||||
class uint256;
|
||||
|
||||
//! -dbcache default (MiB)
|
||||
static const int64_t nDefaultDbCache = 450;
|
||||
//! -dbbatchsize default (bytes)
|
||||
static const int64_t nDefaultDbBatchSize = 16 << 20;
|
||||
//! min. -dbcache (MiB)
|
||||
static const int64_t nMinDbCache = 4;
|
||||
//! Max memory allocated to block tree DB specific cache, if no -txindex (MiB)
|
||||
static const int64_t nMaxBlockDBCache = 2;
|
||||
//! Max memory allocated to block tree DB specific cache, if -txindex (MiB)
|
||||
// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
|
||||
// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
|
||||
static const int64_t nMaxTxIndexCache = 1024;
|
||||
//! Max memory allocated to all block filter index caches combined in MiB.
|
||||
static const int64_t max_filter_index_cache = 1024;
|
||||
//! Max memory allocated to coin DB specific cache (MiB)
|
||||
static const int64_t nMaxCoinsDBCache = 8;
|
||||
|
||||
//! User-controlled performance and debug options.
|
||||
struct CoinsViewOptions {
|
||||
|
|
|
|||
22
src/util/byte_units.h
Normal file
22
src/util/byte_units.h
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
// Copyright (c) 2025-present The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#ifndef BITCOIN_UTIL_BYTE_UNITS_H
|
||||
#define BITCOIN_UTIL_BYTE_UNITS_H
|
||||
|
||||
#include <util/overflow.h>
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
//! Overflow-safe conversion of MiB to bytes.
|
||||
constexpr size_t operator"" _MiB(unsigned long long mebibytes)
|
||||
{
|
||||
auto bytes{CheckedLeftShift(mebibytes, 20)};
|
||||
if (!bytes || *bytes > std::numeric_limits<size_t>::max()) {
|
||||
throw std::overflow_error("MiB value too large for size_t byte conversion");
|
||||
}
|
||||
return *bytes;
|
||||
}
|
||||
|
||||
#endif // BITCOIN_UTIL_BYTE_UNITS_H
|
||||
|
|
@ -5,6 +5,8 @@
|
|||
#ifndef BITCOIN_UTIL_OVERFLOW_H
|
||||
#define BITCOIN_UTIL_OVERFLOW_H
|
||||
|
||||
#include <climits>
|
||||
#include <concepts>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
|
|
@ -47,4 +49,38 @@ template <class T>
|
|||
return i + j;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Left bit shift with overflow checking.
|
||||
* @param input The input value to be left shifted.
|
||||
* @param shift The number of bits to left shift.
|
||||
* @return (input * 2^shift) or nullopt if it would not fit in the return type.
|
||||
*/
|
||||
template <std::integral T>
|
||||
constexpr std::optional<T> CheckedLeftShift(T input, unsigned shift) noexcept
|
||||
{
|
||||
if (shift == 0 || input == 0) return input;
|
||||
// Avoid undefined c++ behaviour if shift is >= number of bits in T.
|
||||
if (shift >= sizeof(T) * CHAR_BIT) return std::nullopt;
|
||||
// If input << shift is too big to fit in T, return nullopt.
|
||||
if (input > (std::numeric_limits<T>::max() >> shift)) return std::nullopt;
|
||||
if (input < (std::numeric_limits<T>::min() >> shift)) return std::nullopt;
|
||||
return input << shift;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Left bit shift with safe minimum and maximum values.
|
||||
* @param input The input value to be left shifted.
|
||||
* @param shift The number of bits to left shift.
|
||||
* @return (input * 2^shift) clamped to fit between the lowest and highest
|
||||
* representable values of the type T.
|
||||
*/
|
||||
template <std::integral T>
|
||||
constexpr T SaturatingLeftShift(T input, unsigned shift) noexcept
|
||||
{
|
||||
if (auto result{CheckedLeftShift(input, shift)}) return *result;
|
||||
// If input << shift is too big to fit in T, return biggest positive or negative
|
||||
// number that fits.
|
||||
return input < 0 ? std::numeric_limits<T>::min() : std::numeric_limits<T>::max();
|
||||
}
|
||||
|
||||
#endif // BITCOIN_UTIL_OVERFLOW_H
|
||||
|
|
|
|||
|
|
@ -1064,11 +1064,11 @@ public:
|
|||
|
||||
//! The total number of bytes available for us to use across all in-memory
|
||||
//! coins caches. This will be split somehow across chainstates.
|
||||
int64_t m_total_coinstip_cache{0};
|
||||
size_t m_total_coinstip_cache{0};
|
||||
//
|
||||
//! The total number of bytes available for us to use across all leveldb
|
||||
//! coins databases. This will be split somehow across chainstates.
|
||||
int64_t m_total_coinsdb_cache{0};
|
||||
size_t m_total_coinsdb_cache{0};
|
||||
|
||||
//! Instantiate a new chainstate.
|
||||
//!
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue