mirrors/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-22 23:07:59 +01:00
Code Issues Projects Releases Wiki Activity

Merge bitcoin/bitcoin#21727: refactor: Move more stuff to blockstorage

Browse source 
fa09a9eac8 style: Add { } to multi-line if (MarcoFalke)
fadafab833 move-only: Move functions to blockstorage (MarcoFalke)
fa7e64d586 move-only: Move constants to blockstorage (MarcoFalke)
fa247a327f refactor: Move block storage globals to blockstorage (MarcoFalke)
fa81c30c6f refactor: Move pruning/reindex/importing globals to blockstorage (MarcoFalke)

Pull request description:

  See #21575

ACKs for top commit:
  Sjors:
    ACK fa09a9eac8
  kiminuo:
    ACK fa09a9e
  laanwj:
    Code review ACK fa09a9eac8
  promag:
    Code review ACK fa09a9eac8. Since last review

Tree-SHA512: 2eb6962ff44da6b77f3058fc02ec66ab742e25ae8dcc8ec62b062896571910d43ca7c4bb16fb3ccb5e5245195b8dec6384b6c8d442fa97ca28d93bdff347d677
This commit is contained in:
W. J. van der Laan 2021-05-05 15:52:29 +02:00
commit 3275c6e578
No known key found for this signature in database
GPG key ID: 1E4AED62986CD25D
8 changed files with 382 additions and 348 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/index/txindex.cpp
View file

@ -4,6 +4,7 @@
#include <index/disktxpos.h>
#include <index/txindex.h>
#include <node/blockstorage.h>
#include <node/ui_interface.h>
#include <shutdown.h>
#include <util/system.h>

41
src/init.cpp
View file

@ -602,47 +602,6 @@ static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex)
}
}
// If we're using -prune with -reindex, then delete block files that will be ignored by the
// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
// is missing, do the same here to delete any later block files after a gap. Also delete all
// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile
// is in sync with what's actually on disk by the time we start downloading, so that pruning
// works correctly.
static void CleanupBlockRevFiles()
{
std::map<std::string, fs::path> mapBlockFiles;
// Glob all blk?????.dat and rev?????.dat files from the blocks directory.
// Remove the rev files immediately and insert the blk file paths into an
// ordered map keyed by block file index.
LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
fs::path blocksdir = gArgs.GetBlocksDirPath();
for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) {
if (fs::is_regular_file(*it) &&
it->path().filename().string().length() == 12 &&
it->path().filename().string().substr(8,4) == ".dat")
{
if (it->path().filename().string().substr(0,3) == "blk")
mapBlockFiles[it->path().filename().string().substr(3,5)] = it->path();
else if (it->path().filename().string().substr(0,3) == "rev")
remove(it->path());
}
}
// Remove all block files that aren't part of a contiguous set starting at
// zero by walking the ordered map (keys are block file indices) by
// keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
// start removing block files.
int nContigCounter = 0;
for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
if (atoi(item.first) == nContigCounter) {
nContigCounter++;
continue;
}
remove(item.second);
}
}
#if HAVE_SYSTEM
static void StartupNotify(const ArgsManager& args)
{

326
src/node/blockstorage.cpp
View file

@ -6,17 +6,310 @@
#include <chain.h>
#include <chainparams.h>
#include <clientversion.h>
#include <consensus/validation.h>
#include <flatfile.h>
#include <fs.h>
#include <hash.h>
#include <pow.h>
#include <shutdown.h>
#include <signet.h>
#include <streams.h>
#include <undo.h>
#include <util/system.h>
#include <validation.h>
// From validation. TODO move here
bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false);
std::atomic_bool fImporting(false);
std::atomic_bool fReindex(false);
bool fHavePruned = false;
bool fPruneMode = false;
uint64_t nPruneTarget = 0;
// TODO make namespace {
RecursiveMutex cs_LastBlockFile;
std::vector<CBlockFileInfo> vinfoBlockFile;
int nLastBlockFile = 0;
/** Global flag to indicate we should check to see if there are
* block/undo files that should be deleted. Set on startup
* or if we allocate more file space when we're in prune mode
*/
bool fCheckForPruning = false;
/** Dirty block index entries. */
std::set<CBlockIndex*> setDirtyBlockIndex;
/** Dirty block file entries. */
std::set<int> setDirtyFileInfo;
// } // namespace
static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false);
static FlatFileSeq BlockFileSeq();
static FlatFileSeq UndoFileSeq();
bool IsBlockPruned(const CBlockIndex* pblockindex)
{
return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0);
}
// If we're using -prune with -reindex, then delete block files that will be ignored by the
// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
// is missing, do the same here to delete any later block files after a gap. Also delete all
// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile
// is in sync with what's actually on disk by the time we start downloading, so that pruning
// works correctly.
void CleanupBlockRevFiles()
{
std::map<std::string, fs::path> mapBlockFiles;
// Glob all blk?????.dat and rev?????.dat files from the blocks directory.
// Remove the rev files immediately and insert the blk file paths into an
// ordered map keyed by block file index.
LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n");
fs::path blocksdir = gArgs.GetBlocksDirPath();
for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) {
if (fs::is_regular_file(*it) &&
it->path().filename().string().length() == 12 &&
it->path().filename().string().substr(8,4) == ".dat")
{
if (it->path().filename().string().substr(0, 3) == "blk") {
mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path();
} else if (it->path().filename().string().substr(0, 3) == "rev") {
remove(it->path());
}
}
}
// Remove all block files that aren't part of a contiguous set starting at
// zero by walking the ordered map (keys are block file indices) by
// keeping a separate counter. Once we hit a gap (or if 0 doesn't exist)
// start removing block files.
int nContigCounter = 0;
for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) {
if (atoi(item.first) == nContigCounter) {
nContigCounter++;
continue;
}
remove(item.second);
}
}
std::string CBlockFileInfo::ToString() const
{
return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast));
}
CBlockFileInfo* GetBlockFileInfo(size_t n)
{
LOCK(cs_LastBlockFile);
return &vinfoBlockFile.at(n);
}
static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
if (fileout.IsNull()) {
return error("%s: OpenUndoFile failed", __func__);
}
// Write index header
unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion());
fileout << messageStart << nSize;
// Write undo data
long fileOutPos = ftell(fileout.Get());
if (fileOutPos < 0) {
return error("%s: ftell failed", __func__);
}
pos.nPos = (unsigned int)fileOutPos;
fileout << blockundo;
// calculate & write checksum
CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION);
hasher << hashBlock;
hasher << blockundo;
fileout << hasher.GetHash();
return true;
}
bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex)
{
FlatFilePos pos = pindex->GetUndoPos();
if (pos.IsNull()) {
return error("%s: no undo data available", __func__);
}
// Open history file to read
CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
return error("%s: OpenUndoFile failed", __func__);
}
// Read block
uint256 hashChecksum;
CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data
try {
verifier << pindex->pprev->GetBlockHash();
verifier >> blockundo;
filein >> hashChecksum;
} catch (const std::exception& e) {
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
}
// Verify checksum
if (hashChecksum != verifier.GetHash()) {
return error("%s: Checksum mismatch", __func__);
}
return true;
}
static void FlushUndoFile(int block_file, bool finalize = false)
{
FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize);
if (!UndoFileSeq().Flush(undo_pos_old, finalize)) {
AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error.");
}
}
void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false)
{
LOCK(cs_LastBlockFile);
FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize);
if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
}
// we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
// e.g. during IBD or a sync after a node going offline
if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo);
}
uint64_t CalculateCurrentUsage()
{
LOCK(cs_LastBlockFile);
uint64_t retval = 0;
for (const CBlockFileInfo& file : vinfoBlockFile) {
retval += file.nSize + file.nUndoSize;
}
return retval;
}
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
{
for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
FlatFilePos pos(*it, 0);
fs::remove(BlockFileSeq().FileName(pos));
fs::remove(UndoFileSeq().FileName(pos));
LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
}
}
static FlatFileSeq BlockFileSeq()
{
return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE);
}
static FlatFileSeq UndoFileSeq()
{
return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE);
}
FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly)
{
return BlockFileSeq().Open(pos, fReadOnly);
}
/** Open an undo file (rev?????.dat) */
static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly)
{
return UndoFileSeq().Open(pos, fReadOnly);
}
fs::path GetBlockPosFilename(const FlatFilePos& pos)
{
return BlockFileSeq().FileName(pos);
}
bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false)
{
LOCK(cs_LastBlockFile);
unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile;
if (vinfoBlockFile.size() <= nFile) {
vinfoBlockFile.resize(nFile + 1);
}
bool finalize_undo = false;
if (!fKnown) {
while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) {
// when the undo file is keeping up with the block file, we want to flush it explicitly
// when it is lagging behind (more blocks arrive than are being connected), we let the
// undo block write case handle it
assert(std::addressof(::ChainActive()) == std::addressof(active_chain));
finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight);
nFile++;
if (vinfoBlockFile.size() <= nFile) {
vinfoBlockFile.resize(nFile + 1);
}
}
pos.nFile = nFile;
pos.nPos = vinfoBlockFile[nFile].nSize;
}
if ((int)nFile != nLastBlockFile) {
if (!fKnown) {
LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString());
}
FlushBlockFile(!fKnown, finalize_undo);
nLastBlockFile = nFile;
}
vinfoBlockFile[nFile].AddBlock(nHeight, nTime);
if (fKnown) {
vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize);
} else {
vinfoBlockFile[nFile].nSize += nAddSize;
}
if (!fKnown) {
bool out_of_space;
size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode("Disk space is too low!", _("Disk space is too low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
}
setDirtyFileInfo.insert(nFile);
return true;
}
static bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize)
{
pos.nFile = nFile;
LOCK(cs_LastBlockFile);
pos.nPos = vinfoBlockFile[nFile].nUndoSize;
vinfoBlockFile[nFile].nUndoSize += nAddSize;
setDirtyFileInfo.insert(nFile);
bool out_of_space;
size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode(state, "Disk space is too low!", _("Disk space is too low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
return true;
}
static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
{
@ -41,6 +334,35 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa
return true;
}
bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams)
{
// Write undo information to disk
if (pindex->GetUndoPos().IsNull()) {
FlatFilePos _pos;
if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) {
return error("ConnectBlock(): FindUndoPos failed");
}
if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) {
return AbortNode(state, "Failed to write undo data");
}
// rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
// we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
// in the block file info as below; note that this does not catch the case where the undo writes are keeping up
// with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
// the FindBlockPos function
if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) {
FlushUndoFile(_pos.nFile, true);
}
// update nUndoPos in block index
pindex->nUndoPos = _pos.nPos;
pindex->nStatus |= BLOCK_HAVE_UNDO;
setDirtyBlockIndex.insert(pindex);
}
return true;
}
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams)
{
block.SetNull();

41
src/node/blockstorage.h
View file

@ -12,7 +12,9 @@
#include <protocol.h> // For CMessageHeader::MessageStartChars
class ArgsManager;
class BlockValidationState;
class CBlock;
class CBlockFileInfo;
class CBlockIndex;
class CBlockUndo;
class CChain;
@ -25,6 +27,44 @@ struct Params;
static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false};
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
extern std::atomic_bool fImporting;
extern std::atomic_bool fReindex;
/** Pruning-related variables and constants */
/** True if any block files have ever been pruned. */
extern bool fHavePruned;
/** True if we're running in -prune mode. */
extern bool fPruneMode;
/** Number of MiB of block files that we're trying to stay below. */
extern uint64_t nPruneTarget;
//! Check whether the block associated with this index entry is pruned or not.
bool IsBlockPruned(const CBlockIndex* pblockindex);
void CleanupBlockRevFiles();
/** Open a block file (blk?????.dat) */
FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false);
/** Translation to a filesystem path */
fs::path GetBlockPosFilename(const FlatFilePos& pos);
/** Get block file info entry for one block file */
CBlockFileInfo* GetBlockFileInfo(size_t n);
/** Calculate the amount of disk space the block & undo files currently use */
uint64_t CalculateCurrentUsage();
/**
* Actually unlink the specified files
*/
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune);
/** Functions for disk access for blocks */
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams);
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams);
@ -32,6 +72,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start);
bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex);
bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams);
FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp);

283
src/validation.cpp
View file

@ -66,10 +66,6 @@
static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000;
/** Maximum kilobytes for transactions to store for processing during reorg */
static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000;
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB
/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */
static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
/** Time to wait between writing blocks/block index to disk. */
static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1};
/** Time to wait between flushing chainstate to disk. */
@ -135,14 +131,9 @@ Mutex g_best_block_mutex;
std::condition_variable g_best_block_cv;
uint256 g_best_block;
bool g_parallel_script_checks{false};
std::atomic_bool fImporting(false);
std::atomic_bool fReindex(false);
bool fHavePruned = false;
bool fPruneMode = false;
bool fRequireStandard = true;
bool fCheckBlockIndex = false;
bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED;
uint64_t nPruneTarget = 0;
int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE;
uint256 hashAssumeValid;
@ -153,22 +144,17 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE);
// Internal stuff
namespace {
CBlockIndex* pindexBestInvalid = nullptr;
} // namespace
RecursiveMutex cs_LastBlockFile;
std::vector<CBlockFileInfo> vinfoBlockFile;
int nLastBlockFile = 0;
/** Global flag to indicate we should check to see if there are
* block/undo files that should be deleted. Set on startup
* or if we allocate more file space when we're in prune mode
*/
bool fCheckForPruning = false;
/** Dirty block index entries. */
std::set<CBlockIndex*> setDirtyBlockIndex;
/** Dirty block file entries. */
std::set<int> setDirtyFileInfo;
} // anon namespace
// Internal stuff from blockstorage ...
extern RecursiveMutex cs_LastBlockFile;
extern std::vector<CBlockFileInfo> vinfoBlockFile;
extern int nLastBlockFile;
extern bool fCheckForPruning;
extern std::set<CBlockIndex*> setDirtyBlockIndex;
extern std::set<int> setDirtyFileInfo;
void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false);
// ... TODO move fully to blockstorage
CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const
{
@ -205,9 +191,6 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state,
bool cacheFullScriptStore, PrecomputedTransactionData& txdata,
std::vector<CScriptCheck>* pvChecks = nullptr)
EXCLUSIVE_LOCKS_REQUIRED(cs_main);
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false);
static FlatFileSeq BlockFileSeq();
static FlatFileSeq UndoFileSeq();
bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags)
{
@ -1462,65 +1445,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state,
return true;
}
static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
if (fileout.IsNull())
return error("%s: OpenUndoFile failed", __func__);
// Write index header
unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion());
fileout << messageStart << nSize;
// Write undo data
long fileOutPos = ftell(fileout.Get());
if (fileOutPos < 0)
return error("%s: ftell failed", __func__);
pos.nPos = (unsigned int)fileOutPos;
fileout << blockundo;
// calculate & write checksum
CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION);
hasher << hashBlock;
hasher << blockundo;
fileout << hasher.GetHash();
return true;
}
bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex)
{
FlatFilePos pos = pindex->GetUndoPos();
if (pos.IsNull()) {
return error("%s: no undo data available", __func__);
}
// Open history file to read
CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION);
if (filein.IsNull())
return error("%s: OpenUndoFile failed", __func__);
// Read block
uint256 hashChecksum;
CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data
try {
verifier << pindex->pprev->GetBlockHash();
verifier >> blockundo;
filein >> hashChecksum;
}
catch (const std::exception& e) {
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
}
// Verify checksum
if (hashChecksum != verifier.GetHash())
return error("%s: Checksum mismatch", __func__);
return true;
}
static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str())
bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage)
{
AbortNode(strMessage, userMessage);
return state.Error(strMessage);
@ -1620,55 +1545,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI
return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN;
}
static void FlushUndoFile(int block_file, bool finalize = false)
{
FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize);
if (!UndoFileSeq().Flush(undo_pos_old, finalize)) {
AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error.");
}
}
static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false)
{
LOCK(cs_LastBlockFile);
FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize);
if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) {
AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
}
// we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks,
// e.g. during IBD or a sync after a node going offline
if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo);
}
static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize);
static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams)
{
// Write undo information to disk
if (pindex->GetUndoPos().IsNull()) {
FlatFilePos _pos;
if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40))
return error("ConnectBlock(): FindUndoPos failed");
if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart()))
return AbortNode(state, "Failed to write undo data");
// rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
// we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
// in the block file info as below; note that this does not catch the case where the undo writes are keeping up
// with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
// the FindBlockPos function
if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) {
FlushUndoFile(_pos.nFile, true);
}
// update nUndoPos in block index
pindex->nUndoPos = _pos.nPos;
pindex->nStatus |= BLOCK_HAVE_UNDO;
setDirtyBlockIndex.insert(pindex);
}
return true;
}
static CCheckQueue<CScriptCheck> scriptcheckqueue(128);
void StartScriptCheckWorkerThreads(int threads_num)
@ -3102,84 +2978,6 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi
}
}
// TODO move to blockstorage
bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false)
{
LOCK(cs_LastBlockFile);
unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile;
if (vinfoBlockFile.size() <= nFile) {
vinfoBlockFile.resize(nFile + 1);
}
bool finalize_undo = false;
if (!fKnown) {
while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) {
// when the undo file is keeping up with the block file, we want to flush it explicitly
// when it is lagging behind (more blocks arrive than are being connected), we let the
// undo block write case handle it
assert(std::addressof(::ChainActive()) == std::addressof(active_chain));
finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight);
nFile++;
if (vinfoBlockFile.size() <= nFile) {
vinfoBlockFile.resize(nFile + 1);
}
}
pos.nFile = nFile;
pos.nPos = vinfoBlockFile[nFile].nSize;
}
if ((int)nFile != nLastBlockFile) {
if (!fKnown) {
LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString());
}
FlushBlockFile(!fKnown, finalize_undo);
nLastBlockFile = nFile;
}
vinfoBlockFile[nFile].AddBlock(nHeight, nTime);
if (fKnown)
vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize);
else
vinfoBlockFile[nFile].nSize += nAddSize;
if (!fKnown) {
bool out_of_space;
size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode("Disk space is too low!", _("Disk space is too low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
}
setDirtyFileInfo.insert(nFile);
return true;
}
static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
{
pos.nFile = nFile;
LOCK(cs_LastBlockFile);
pos.nPos = vinfoBlockFile[nFile].nUndoSize;
vinfoBlockFile[nFile].nUndoSize += nAddSize;
setDirtyFileInfo.insert(nFile);
bool out_of_space;
size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode(state, "Disk space is too low!", _("Disk space is too low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
return true;
}
static bool CheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true)
{
// Check proof of work matches claimed amount
@ -3737,18 +3535,6 @@ bool TestBlockValidity(BlockValidationState& state,
* BLOCK PRUNING CODE
*/
/* Calculate the amount of disk space the block & undo files currently use */
uint64_t CalculateCurrentUsage()
{
LOCK(cs_LastBlockFile);
uint64_t retval = 0;
for (const CBlockFileInfo &file : vinfoBlockFile) {
retval += file.nSize + file.nUndoSize;
}
return retval;
}
void BlockManager::PruneOneBlockFile(const int fileNumber)
{
AssertLockHeld(cs_main);
@ -3783,17 +3569,6 @@ void BlockManager::PruneOneBlockFile(const int fileNumber)
setDirtyFileInfo.insert(fileNumber);
}
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
{
for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
FlatFilePos pos(*it, 0);
fs::remove(BlockFileSeq().FileName(pos));
fs::remove(UndoFileSeq().FileName(pos));
LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
}
}
void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height)
{
assert(fPruneMode && nManualPruneHeight > 0);
@ -3888,30 +3663,6 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr
nLastBlockWeCanPrune, count);
}
static FlatFileSeq BlockFileSeq()
{
return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE);
}
static FlatFileSeq UndoFileSeq()
{
return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE);
}
FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) {
return BlockFileSeq().Open(pos, fReadOnly);
}
/** Open an undo file (rev?????.dat) */
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) {
return UndoFileSeq().Open(pos, fReadOnly);
}
fs::path GetBlockPosFilename(const FlatFilePos &pos)
{
return BlockFileSeq().FileName(pos);
}
CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash)
{
AssertLockHeld(cs_main);
@ -4744,18 +4495,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size)
return ret;
}
std::string CBlockFileInfo::ToString() const
{
return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast));
}
CBlockFileInfo* GetBlockFileInfo(size_t n)
{
LOCK(cs_LastBlockFile);
return &vinfoBlockFile.at(n);
}
static const uint64_t MEMPOOL_DUMP_VERSION = 1;
bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function)

35
src/validation.h
View file

@ -27,6 +27,7 @@
#include <serialize.h>
#include <util/check.h>
#include <util/hasher.h>
#include <util/translation.h>
#include <atomic>
#include <map>
@ -70,8 +71,6 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25;
static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101;
/** Default for -mempoolexpiry, expiration time for mempool transactions in hours */
static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336;
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
/** Maximum number of dedicated script-checking threads allowed */
static const int MAX_SCRIPTCHECK_THREADS = 15;
/** -par default (number of script-checking threads, 0 = auto) */
@ -113,8 +112,6 @@ typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap;
extern Mutex g_best_block_mutex;
extern std::condition_variable g_best_block_cv;
extern uint256 g_best_block;
extern std::atomic_bool fImporting;
extern std::atomic_bool fReindex;
/** Whether there are dedicated script-checking threads running.
* False indicates all script checking is done on the main threadMessageHandler thread.
*/
@ -136,20 +133,9 @@ extern arith_uint256 nMinimumChainWork;
/** Best header we've seen so far (used for getheaders queries' starting points). */
extern CBlockIndex *pindexBestHeader;
/** Pruning-related variables and constants */
/** True if any block files have ever been pruned. */
extern bool fHavePruned;
/** True if we're running in -prune mode. */
extern bool fPruneMode;
/** Number of MiB of block files that we're trying to stay below. */
extern uint64_t nPruneTarget;
/** Documentation for argument 'checklevel'. */
extern const std::vector<std::string> CHECKLEVEL_DOC;
/** Open a block file (blk?????.dat) */
FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false);
/** Translation to a filesystem path */
fs::path GetBlockPosFilename(const FlatFilePos &pos);
/** Unload database information */
void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman);
/** Run instances of script checking worker threads */
@ -171,17 +157,11 @@ void StopScriptCheckWorkerThreads();
CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock);
CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams);
bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str{});
/** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */
double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex);
/** Calculate the amount of disk space the block & undo files currently use */
uint64_t CalculateCurrentUsage();
/**
* Actually unlink the specified files
*/
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune);
/** Prune block files up to a given height */
void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight);
@ -1009,9 +989,6 @@ extern VersionBitsCache versionbitscache;
*/
int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params);
/** Get block file info entry for one block file */
CBlockFileInfo* GetBlockFileInfo(size_t n);
using FopenFn = std::function<FILE*(const fs::path&, const char*)>;
/** Dump the mempool to disk. */
@ -1020,12 +997,6 @@ bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbri
/** Load the mempool from disk. */
bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen);
//! Check whether the block associated with this index entry is pruned or not.
inline bool IsBlockPruned(const CBlockIndex* pblockindex)
{
return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0);
}
/**
* Return the expected assumeutxo value for a given height, if one exists.
*

1
src/wallet/test/wallet_tests.cpp
View file

@ -11,6 +11,7 @@
#include <vector>
#include <interfaces/chain.h>
#include <node/blockstorage.h>
#include <node/context.h>
#include <policy/policy.h>
#include <rpc/server.h>

2
test/lint/lint-locale-dependence.sh
View file

@ -43,7 +43,7 @@ KNOWN_VIOLATIONS=(
"src/dbwrapper.cpp.*stoul"
"src/dbwrapper.cpp:.*vsnprintf"
"src/httprpc.cpp.*trim"
"src/init.cpp:.*atoi"
"src/node/blockstorage.cpp:.*atoi"
"src/qt/rpcconsole.cpp:.*atoi"
"src/rest.cpp:.*strtol"
"src/test/dbwrapper_tests.cpp:.*snprintf"