mirror of
https://github.com/bitcoin/bitcoin.git
synced 2024-11-20 02:25:40 +01:00
Merge #13033: Build txindex in parallel with validation
9b2704777c[doc] Include txindex changes in the release notes. (Jim Posen)ed77dd6b30[test] Simple unit test for TxIndex. (Jim Posen)6d772a3d44[rpc] Public interfaces to GetTransaction block until synced. (Jim Posen)a03f804f2a[index] Move disk IO logic from GetTransaction to TxIndex::FindTx. (Jim Posen)e0a3b80033[validation] Replace tx index code in validation code with TxIndex. (Jim Posen)8181db88f6[init] Initialize and start TxIndex in init code. (Jim Posen)f90c3a62f5[index] TxIndex method to wait until caught up. (Jim Posen)70d510d93c[index] Allow TxIndex sync thread to be interrupted. (Jim Posen)94b4f8bbb9[index] TxIndex initial sync thread. (Jim Posen)34d68bf3a3[index] Create new TxIndex class. (Jim Posen)c88bcec93f[db] Migration for txindex data to new, separate database. (Jim Posen)0cb8303241[db] Create separate database for txindex. (Jim Posen) Pull request description: I'm re-opening #11857 as a new pull request because the last one stopped loading for people ------------------------------- This refactors the tx index code to be in it's own class and get built concurrently with validation code. The main benefit is decoupling and moving the txindex into a separate DB. The primary motivation is to lay the groundwork for other indexers that might be desired (such as the [compact filters](https://github.com/bitcoin/bips/pull/636)). The basic idea is that the TxIndex spins up its own thread, which first syncs the txindex to the current block index, then once in sync the BlockConnected ValidationInterface hook writes new blocks. ### DB changes At the suggestion of some other developers, the txindex has been split out into a separate database. A data migration runs at startup on any nodes with a legacy txindex. Currently the migration blocks node initialization until complete. ### Open questions - Should the migration of txindex data from the old DB to the new DB block in init or should it happen in a background thread? The downside to backgrounding it is that `getrawtransaction` would return an error message saying the txindex is syncing while the migration is running. ### Impact In a sample size n=1 test where I synced nodes from scratch, the average time [Index writing](https://github.com/bitcoin/bitcoin/blob/master/src/validation.cpp#L1903) was 3.36ms in master and 1.72ms in this branch. The average time between `UpdateTip` log lines for sequential blocks between 400,000 and IBD end on mainnet was 0.297204s in master and 0.286134s in this branch. Most likely this is just variance in IBD times, but I can try with some more trials if people want. Tree-SHA512: 451fd7d95df89dfafceaa723cdf0f7b137615b531cf5c5035cfb54e9ccc2026cec5ac85edbcf71b7f4e2f102e36e9202b8b3a667e1504a9e1a9976ab1f0079c4
This commit is contained in:
Pieter Wuille
commit
a07e8caa5d
@ -10,6 +10,7 @@
|
||||
* db.log: wallet database log file; moved to wallets/ directory on new installs since 0.16.0
|
||||
* debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt
|
||||
* fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0
|
||||
* indexes/txindex/*: optional transaction index database (LevelDB); since 0.17.0
|
||||
* mempool.dat: dump of the mempool's transactions; since 0.14.0.
|
||||
* peers.dat: peer IP address database (custom format); since 0.7.0
|
||||
* wallet.dat: personal wallet (BDB) with keys and transactions; moved to wallets/ directory on new installs since 0.16.0
|
||||
|
||||
11
doc/release-notes-pr13033.md
Normal file
11
doc/release-notes-pr13033.md
Normal file
@ -0,0 +1,11 @@
|
||||
Transaction index changes
|
||||
-------------------------
|
||||
|
||||
The transaction index is now built separately from the main node procedure,
|
||||
meaning the `-txindex` flag can be toggled without a full reindex. If bitcoind
|
||||
is run with `-txindex` on a node that is already partially or fully synced
|
||||
without one, the transaction index will be built in the background and become
|
||||
available once caught up. When switching from running `-txindex` to running
|
||||
without the flag, the transaction index database will *not* be deleted
|
||||
automatically, meaning it could be turned back on at a later time without a full
|
||||
resync.
|
||||
@ -103,6 +103,7 @@ BITCOIN_CORE_H = \
|
||||
fs.h \
|
||||
httprpc.h \
|
||||
httpserver.h \
|
||||
index/txindex.h \
|
||||
indirectmap.h \
|
||||
init.h \
|
||||
interfaces/handler.h \
|
||||
@ -204,6 +205,7 @@ libbitcoin_server_a_SOURCES = \
|
||||
consensus/tx_verify.cpp \
|
||||
httprpc.cpp \
|
||||
httpserver.cpp \
|
||||
index/txindex.cpp \
|
||||
init.cpp \
|
||||
dbwrapper.cpp \
|
||||
merkleblock.cpp \
|
||||
|
||||
@ -83,6 +83,7 @@ BITCOIN_TESTS =\
|
||||
test/timedata_tests.cpp \
|
||||
test/torcontrol_tests.cpp \
|
||||
test/transaction_tests.cpp \
|
||||
test/txindex_tests.cpp \
|
||||
test/txvalidation_tests.cpp \
|
||||
test/txvalidationcache_tests.cpp \
|
||||
test/versionbits_tests.cpp \
|
||||
|
||||
@ -224,6 +224,9 @@ public:
|
||||
CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
|
||||
~CDBWrapper();
|
||||
|
||||
CDBWrapper(const CDBWrapper&) = delete;
|
||||
CDBWrapper& operator=(const CDBWrapper&) = delete;
|
||||
|
||||
template <typename K, typename V>
|
||||
bool Read(const K& key, V& value) const
|
||||
{
|
||||
|
||||
309
src/index/txindex.cpp
Normal file
309
src/index/txindex.cpp
Normal file
@ -0,0 +1,309 @@
|
||||
// Copyright (c) 2017-2018 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include <chainparams.h>
|
||||
#include <index/txindex.h>
|
||||
#include <init.h>
|
||||
#include <tinyformat.h>
|
||||
#include <ui_interface.h>
|
||||
#include <util.h>
|
||||
#include <validation.h>
|
||||
#include <warnings.h>
|
||||
|
||||
constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds
|
||||
constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds
|
||||
|
||||
std::unique_ptr<TxIndex> g_txindex;
|
||||
|
||||
template<typename... Args>
|
||||
static void FatalError(const char* fmt, const Args&... args)
|
||||
{
|
||||
std::string strMessage = tfm::format(fmt, args...);
|
||||
SetMiscWarning(strMessage);
|
||||
LogPrintf("*** %s\n", strMessage);
|
||||
uiInterface.ThreadSafeMessageBox(
|
||||
"Error: A fatal internal error occurred, see debug.log for details",
|
||||
"", CClientUIInterface::MSG_ERROR);
|
||||
StartShutdown();
|
||||
}
|
||||
|
||||
TxIndex::TxIndex(std::unique_ptr<TxIndexDB> db) :
|
||||
m_db(std::move(db)), m_synced(false), m_best_block_index(nullptr)
|
||||
{}
|
||||
|
||||
TxIndex::~TxIndex()
|
||||
{
|
||||
Interrupt();
|
||||
Stop();
|
||||
}
|
||||
|
||||
bool TxIndex::Init()
|
||||
{
|
||||
LOCK(cs_main);
|
||||
|
||||
// Attempt to migrate txindex from the old database to the new one. Even if
|
||||
// chain_tip is null, the node could be reindexing and we still want to
|
||||
// delete txindex records in the old database.
|
||||
if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
CBlockLocator locator;
|
||||
if (!m_db->ReadBestBlock(locator)) {
|
||||
locator.SetNull();
|
||||
}
|
||||
|
||||
m_best_block_index = FindForkInGlobalIndex(chainActive, locator);
|
||||
m_synced = m_best_block_index.load() == chainActive.Tip();
|
||||
return true;
|
||||
}
|
||||
|
||||
static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev)
|
||||
{
|
||||
AssertLockHeld(cs_main);
|
||||
|
||||
if (!pindex_prev) {
|
||||
return chainActive.Genesis();
|
||||
}
|
||||
|
||||
const CBlockIndex* pindex = chainActive.Next(pindex_prev);
|
||||
if (pindex) {
|
||||
return pindex;
|
||||
}
|
||||
|
||||
return chainActive.Next(chainActive.FindFork(pindex_prev));
|
||||
}
|
||||
|
||||
void TxIndex::ThreadSync()
|
||||
{
|
||||
const CBlockIndex* pindex = m_best_block_index.load();
|
||||
if (!m_synced) {
|
||||
auto& consensus_params = Params().GetConsensus();
|
||||
|
||||
int64_t last_log_time = 0;
|
||||
int64_t last_locator_write_time = 0;
|
||||
while (true) {
|
||||
if (m_interrupt) {
|
||||
WriteBestBlock(pindex);
|
||||
return;
|
||||
}
|
||||
|
||||
{
|
||||
LOCK(cs_main);
|
||||
const CBlockIndex* pindex_next = NextSyncBlock(pindex);
|
||||
if (!pindex_next) {
|
||||
WriteBestBlock(pindex);
|
||||
m_best_block_index = pindex;
|
||||
m_synced = true;
|
||||
break;
|
||||
}
|
||||
pindex = pindex_next;
|
||||
}
|
||||
|
||||
int64_t current_time = GetTime();
|
||||
if (last_log_time + SYNC_LOG_INTERVAL < current_time) {
|
||||
LogPrintf("Syncing txindex with block chain from height %d\n", pindex->nHeight);
|
||||
last_log_time = current_time;
|
||||
}
|
||||
|
||||
if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
|
||||
WriteBestBlock(pindex);
|
||||
last_locator_write_time = current_time;
|
||||
}
|
||||
|
||||
CBlock block;
|
||||
if (!ReadBlockFromDisk(block, pindex, consensus_params)) {
|
||||
FatalError("%s: Failed to read block %s from disk",
|
||||
__func__, pindex->GetBlockHash().ToString());
|
||||
return;
|
||||
}
|
||||
if (!WriteBlock(block, pindex)) {
|
||||
FatalError("%s: Failed to write block %s to tx index database",
|
||||
__func__, pindex->GetBlockHash().ToString());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (pindex) {
|
||||
LogPrintf("txindex is enabled at height %d\n", pindex->nHeight);
|
||||
} else {
|
||||
LogPrintf("txindex is enabled\n");
|
||||
}
|
||||
}
|
||||
|
||||
bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
|
||||
{
|
||||
CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
|
||||
std::vector<std::pair<uint256, CDiskTxPos>> vPos;
|
||||
vPos.reserve(block.vtx.size());
|
||||
for (const auto& tx : block.vtx) {
|
||||
vPos.emplace_back(tx->GetHash(), pos);
|
||||
pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION);
|
||||
}
|
||||
return m_db->WriteTxs(vPos);
|
||||
}
|
||||
|
||||
bool TxIndex::WriteBestBlock(const CBlockIndex* block_index)
|
||||
{
|
||||
LOCK(cs_main);
|
||||
if (!m_db->WriteBestBlock(chainActive.GetLocator(block_index))) {
|
||||
return error("%s: Failed to write locator to disk", __func__);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void TxIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
|
||||
const std::vector<CTransactionRef>& txn_conflicted)
|
||||
{
|
||||
if (!m_synced) {
|
||||
return;
|
||||
}
|
||||
|
||||
const CBlockIndex* best_block_index = m_best_block_index.load();
|
||||
if (!best_block_index) {
|
||||
if (pindex->nHeight != 0) {
|
||||
FatalError("%s: First block connected is not the genesis block (height=%d)",
|
||||
__func__, pindex->nHeight);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Ensure block connects to an ancestor of the current best block. This should be the case
|
||||
// most of the time, but may not be immediately after the the sync thread catches up and sets
|
||||
// m_synced. Consider the case where there is a reorg and the blocks on the stale branch are
|
||||
// in the ValidationInterface queue backlog even after the sync thread has caught up to the
|
||||
// new chain tip. In this unlikely event, log a warning and let the queue clear.
|
||||
if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) {
|
||||
LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */
|
||||
"known best chain (tip=%s); not updating txindex\n",
|
||||
__func__, pindex->GetBlockHash().ToString(),
|
||||
best_block_index->GetBlockHash().ToString());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (WriteBlock(*block, pindex)) {
|
||||
m_best_block_index = pindex;
|
||||
} else {
|
||||
FatalError("%s: Failed to write block %s to txindex",
|
||||
__func__, pindex->GetBlockHash().ToString());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void TxIndex::SetBestChain(const CBlockLocator& locator)
|
||||
{
|
||||
if (!m_synced) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint256& locator_tip_hash = locator.vHave.front();
|
||||
const CBlockIndex* locator_tip_index;
|
||||
{
|
||||
LOCK(cs_main);
|
||||
locator_tip_index = LookupBlockIndex(locator_tip_hash);
|
||||
}
|
||||
|
||||
if (!locator_tip_index) {
|
||||
FatalError("%s: First block (hash=%s) in locator was not found",
|
||||
__func__, locator_tip_hash.ToString());
|
||||
return;
|
||||
}
|
||||
|
||||
// This checks that SetBestChain callbacks are received after BlockConnected. The check may fail
|
||||
// immediately after the the sync thread catches up and sets m_synced. Consider the case where
|
||||
// there is a reorg and the blocks on the stale branch are in the ValidationInterface queue
|
||||
// backlog even after the sync thread has caught up to the new chain tip. In this unlikely
|
||||
// event, log a warning and let the queue clear.
|
||||
const CBlockIndex* best_block_index = m_best_block_index.load();
|
||||
if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) {
|
||||
LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */
|
||||
"chain (tip=%s); not writing txindex locator\n",
|
||||
__func__, locator_tip_hash.ToString(),
|
||||
best_block_index->GetBlockHash().ToString());
|
||||
return;
|
||||
}
|
||||
|
||||
if (!m_db->WriteBestBlock(locator)) {
|
||||
error("%s: Failed to write locator to disk", __func__);
|
||||
}
|
||||
}
|
||||
|
||||
bool TxIndex::BlockUntilSyncedToCurrentChain()
|
||||
{
|
||||
AssertLockNotHeld(cs_main);
|
||||
|
||||
if (!m_synced) {
|
||||
return false;
|
||||
}
|
||||
|
||||
{
|
||||
// Skip the queue-draining stuff if we know we're caught up with
|
||||
// chainActive.Tip().
|
||||
LOCK(cs_main);
|
||||
const CBlockIndex* chain_tip = chainActive.Tip();
|
||||
const CBlockIndex* best_block_index = m_best_block_index.load();
|
||||
if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
LogPrintf("%s: txindex is catching up on block notifications\n", __func__);
|
||||
SyncWithValidationInterfaceQueue();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
|
||||
{
|
||||
CDiskTxPos postx;
|
||||
if (!m_db->ReadTxPos(tx_hash, postx)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
|
||||
if (file.IsNull()) {
|
||||
return error("%s: OpenBlockFile failed", __func__);
|
||||
}
|
||||
CBlockHeader header;
|
||||
try {
|
||||
file >> header;
|
||||
fseek(file.Get(), postx.nTxOffset, SEEK_CUR);
|
||||
file >> tx;
|
||||
} catch (const std::exception& e) {
|
||||
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
|
||||
}
|
||||
if (tx->GetHash() != tx_hash) {
|
||||
return error("%s: txid mismatch", __func__);
|
||||
}
|
||||
block_hash = header.GetHash();
|
||||
return true;
|
||||
}
|
||||
|
||||
void TxIndex::Interrupt()
|
||||
{
|
||||
m_interrupt();
|
||||
}
|
||||
|
||||
void TxIndex::Start()
|
||||
{
|
||||
// Need to register this ValidationInterface before running Init(), so that
|
||||
// callbacks are not missed if Init sets m_synced to true.
|
||||
RegisterValidationInterface(this);
|
||||
if (!Init()) {
|
||||
FatalError("%s: txindex failed to initialize", __func__);
|
||||
return;
|
||||
}
|
||||
|
||||
m_thread_sync = std::thread(&TraceThread<std::function<void()>>, "txindex",
|
||||
std::bind(&TxIndex::ThreadSync, this));
|
||||
}
|
||||
|
||||
void TxIndex::Stop()
|
||||
{
|
||||
UnregisterValidationInterface(this);
|
||||
|
||||
if (m_thread_sync.joinable()) {
|
||||
m_thread_sync.join();
|
||||
}
|
||||
}
|
||||
94
src/index/txindex.h
Normal file
94
src/index/txindex.h
Normal file
@ -0,0 +1,94 @@
|
||||
// Copyright (c) 2017-2018 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#ifndef BITCOIN_INDEX_TXINDEX_H
|
||||
#define BITCOIN_INDEX_TXINDEX_H
|
||||
|
||||
#include <primitives/block.h>
|
||||
#include <primitives/transaction.h>
|
||||
#include <threadinterrupt.h>
|
||||
#include <txdb.h>
|
||||
#include <uint256.h>
|
||||
#include <validationinterface.h>
|
||||
|
||||
class CBlockIndex;
|
||||
|
||||
/**
|
||||
* TxIndex is used to look up transactions included in the blockchain by hash.
|
||||
* The index is written to a LevelDB database and records the filesystem
|
||||
* location of each transaction by transaction hash.
|
||||
*/
|
||||
class TxIndex final : public CValidationInterface
|
||||
{
|
||||
private:
|
||||
const std::unique_ptr<TxIndexDB> m_db;
|
||||
|
||||
/// Whether the index is in sync with the main chain. The flag is flipped
|
||||
/// from false to true once, after which point this starts processing
|
||||
/// ValidationInterface notifications to stay in sync.
|
||||
std::atomic<bool> m_synced;
|
||||
|
||||
/// The last block in the chain that the TxIndex is in sync with.
|
||||
std::atomic<const CBlockIndex*> m_best_block_index;
|
||||
|
||||
std::thread m_thread_sync;
|
||||
CThreadInterrupt m_interrupt;
|
||||
|
||||
/// Initialize internal state from the database and block index.
|
||||
bool Init();
|
||||
|
||||
/// Sync the tx index with the block index starting from the current best
|
||||
/// block. Intended to be run in its own thread, m_thread_sync, and can be
|
||||
/// interrupted with m_interrupt. Once the txindex gets in sync, the
|
||||
/// m_synced flag is set and the BlockConnected ValidationInterface callback
|
||||
/// takes over and the sync thread exits.
|
||||
void ThreadSync();
|
||||
|
||||
/// Write update index entries for a newly connected block.
|
||||
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex);
|
||||
|
||||
/// Write the current chain block locator to the DB.
|
||||
bool WriteBestBlock(const CBlockIndex* block_index);
|
||||
|
||||
protected:
|
||||
void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
|
||||
const std::vector<CTransactionRef>& txn_conflicted) override;
|
||||
|
||||
void SetBestChain(const CBlockLocator& locator) override;
|
||||
|
||||
public:
|
||||
/// Constructs the TxIndex, which becomes available to be queried.
|
||||
explicit TxIndex(std::unique_ptr<TxIndexDB> db);
|
||||
|
||||
/// Destructor interrupts sync thread if running and blocks until it exits.
|
||||
~TxIndex();
|
||||
|
||||
/// Blocks the current thread until the transaction index is caught up to
|
||||
/// the current state of the block chain. This only blocks if the index has gotten in sync once
|
||||
/// and only needs to process blocks in the ValidationInterface queue. If the index is catching
|
||||
/// up from far behind, this method does not block and immediately returns false.
|
||||
bool BlockUntilSyncedToCurrentChain();
|
||||
|
||||
/// Look up a transaction by hash.
|
||||
///
|
||||
/// @param[in] tx_hash The hash of the transaction to be returned.
|
||||
/// @param[out] block_hash The hash of the block the transaction is found in.
|
||||
/// @param[out] tx The transaction itself.
|
||||
/// @return true if transaction is found, false otherwise
|
||||
bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const;
|
||||
|
||||
void Interrupt();
|
||||
|
||||
/// Start initializes the sync state and registers the instance as a
|
||||
/// ValidationInterface so that it stays in sync with blockchain updates.
|
||||
void Start();
|
||||
|
||||
/// Stops the instance from staying in sync with blockchain updates.
|
||||
void Stop();
|
||||
};
|
||||
|
||||
/// The global transaction index, used in GetTransaction. May be null.
|
||||
extern std::unique_ptr<TxIndex> g_txindex;
|
||||
|
||||
#endif // BITCOIN_INDEX_TXINDEX_H
|
||||
43
src/init.cpp
43
src/init.cpp
@ -19,6 +19,7 @@
|
||||
#include <fs.h>
|
||||
#include <httpserver.h>
|
||||
#include <httprpc.h>
|
||||
#include <index/txindex.h>
|
||||
#include <key.h>
|
||||
#include <validation.h>
|
||||
#include <miner.h>
|
||||
@ -182,6 +183,9 @@ void Interrupt()
|
||||
InterruptMapPort();
|
||||
if (g_connman)
|
||||
g_connman->Interrupt();
|
||||
if (g_txindex) {
|
||||
g_txindex->Interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
void Shutdown()
|
||||
@ -212,6 +216,9 @@ void Shutdown()
|
||||
if (g_connman) g_connman->Stop();
|
||||
peerLogic.reset();
|
||||
g_connman.reset();
|
||||
if (g_txindex) {
|
||||
g_txindex.reset();
|
||||
}
|
||||
|
||||
StopTorControl();
|
||||
|
||||
@ -1414,9 +1421,10 @@ bool AppInitMain()
|
||||
int64_t nTotalCache = (gArgs.GetArg("-dbcache", nDefaultDbCache) << 20);
|
||||
nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
|
||||
nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache
|
||||
int64_t nBlockTreeDBCache = nTotalCache / 8;
|
||||
nBlockTreeDBCache = std::min(nBlockTreeDBCache, (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxBlockDBAndTxIndexCache : nMaxBlockDBCache) << 20);
|
||||
int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
|
||||
nTotalCache -= nBlockTreeDBCache;
|
||||
int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
|
||||
nTotalCache -= nTxIndexCache;
|
||||
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
|
||||
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
|
||||
nTotalCache -= nCoinDBCache;
|
||||
@ -1424,6 +1432,9 @@ bool AppInitMain()
|
||||
int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
|
||||
LogPrintf("Cache configuration:\n");
|
||||
LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
|
||||
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
|
||||
LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
|
||||
}
|
||||
LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
|
||||
LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
|
||||
|
||||
@ -1457,9 +1468,8 @@ bool AppInitMain()
|
||||
|
||||
if (fRequestShutdown) break;
|
||||
|
||||
// LoadBlockIndex will load fTxIndex from the db, or set it if
|
||||
// we're reindexing. It will also load fHavePruned if we've
|
||||
// ever removed a block file from disk.
|
||||
// LoadBlockIndex will load fHavePruned if we've ever removed a
|
||||
// block file from disk.
|
||||
// Note that it also sets fReindex based on the disk flag!
|
||||
// From here on out fReindex and fReset mean something different!
|
||||
if (!LoadBlockIndex(chainparams)) {
|
||||
@ -1473,12 +1483,6 @@ bool AppInitMain()
|
||||
return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?"));
|
||||
}
|
||||
|
||||
// Check for changed -txindex state
|
||||
if (fTxIndex != gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
|
||||
strLoadError = _("You need to rebuild the database using -reindex to change -txindex");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for changed -prune state. What we are concerned about is a user who has pruned blocks
|
||||
// in the past, but is now trying to run unpruned.
|
||||
if (fHavePruned && !fPruneMode) {
|
||||
@ -1608,10 +1612,17 @@ bool AppInitMain()
|
||||
::feeEstimator.Read(est_filein);
|
||||
fFeeEstimatesInitialized = true;
|
||||
|
||||
// ********************************************************* Step 8: load wallet
|
||||
// ********************************************************* Step 8: start indexers
|
||||
if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
|
||||
auto txindex_db = MakeUnique<TxIndexDB>(nTxIndexCache, false, fReindex);
|
||||
g_txindex = MakeUnique<TxIndex>(std::move(txindex_db));
|
||||
g_txindex->Start();
|
||||
}
|
||||
|
||||
// ********************************************************* Step 9: load wallet
|
||||
if (!g_wallet_init_interface.Open()) return false;
|
||||
|
||||
// ********************************************************* Step 9: data directory maintenance
|
||||
// ********************************************************* Step 10: data directory maintenance
|
||||
|
||||
// if pruning, unset the service bit and perform the initial blockstore prune
|
||||
// after any wallet rescanning has taken place.
|
||||
@ -1633,7 +1644,7 @@ bool AppInitMain()
|
||||
nLocalServices = ServiceFlags(nLocalServices | NODE_WITNESS);
|
||||
}
|
||||
|
||||
// ********************************************************* Step 10: import blocks
|
||||
// ********************************************************* Step 11: import blocks
|
||||
|
||||
if (!CheckDiskSpace() && !CheckDiskSpace(0, true))
|
||||
return false;
|
||||
@ -1672,7 +1683,7 @@ bool AppInitMain()
|
||||
return false;
|
||||
}
|
||||
|
||||
// ********************************************************* Step 11: start node
|
||||
// ********************************************************* Step 12: start node
|
||||
|
||||
int chain_active_height;
|
||||
|
||||
@ -1750,7 +1761,7 @@ bool AppInitMain()
|
||||
return false;
|
||||
}
|
||||
|
||||
// ********************************************************* Step 12: finished
|
||||
// ********************************************************* Step 13: finished
|
||||
|
||||
SetRPCWarmupFinished();
|
||||
uiInterface.InitMessage(_("Done loading"));
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
#include <chain.h>
|
||||
#include <chainparams.h>
|
||||
#include <core_io.h>
|
||||
#include <index/txindex.h>
|
||||
#include <primitives/block.h>
|
||||
#include <primitives/transaction.h>
|
||||
#include <validation.h>
|
||||
@ -350,6 +351,10 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart)
|
||||
if (!ParseHashStr(hashStr, hash))
|
||||
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr);
|
||||
|
||||
if (g_txindex) {
|
||||
g_txindex->BlockUntilSyncedToCurrentChain();
|
||||
}
|
||||
|
||||
CTransactionRef tx;
|
||||
uint256 hashBlock = uint256();
|
||||
if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true))
|
||||
|
||||
@ -47,8 +47,6 @@ static std::mutex cs_blockchange;
|
||||
static std::condition_variable cond_blockchange;
|
||||
static CUpdatedBlock latestblock;
|
||||
|
||||
extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry);
|
||||
|
||||
/* Calculate the difficulty for a given block index,
|
||||
* or the block index of the given chain.
|
||||
*/
|
||||
|
||||
@ -7,6 +7,7 @@
|
||||
#include <coins.h>
|
||||
#include <consensus/validation.h>
|
||||
#include <core_io.h>
|
||||
#include <index/txindex.h>
|
||||
#include <init.h>
|
||||
#include <keystore.h>
|
||||
#include <validation.h>
|
||||
@ -47,6 +48,8 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry)
|
||||
TxToUniv(tx, uint256(), entry, true, RPCSerializationFlags());
|
||||
|
||||
if (!hashBlock.IsNull()) {
|
||||
LOCK(cs_main);
|
||||
|
||||
entry.pushKV("blockhash", hashBlock.GetHex());
|
||||
CBlockIndex* pindex = LookupBlockIndex(hashBlock);
|
||||
if (pindex) {
|
||||
@ -141,8 +144,6 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
|
||||
+ HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"")
|
||||
);
|
||||
|
||||
LOCK(cs_main);
|
||||
|
||||
bool in_active_chain = true;
|
||||
uint256 hash = ParseHashV(request.params[0], "parameter 1");
|
||||
CBlockIndex* blockindex = nullptr;
|
||||
@ -159,6 +160,8 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
|
||||
}
|
||||
|
||||
if (!request.params[2].isNull()) {
|
||||
LOCK(cs_main);
|
||||
|
||||
uint256 blockhash = ParseHashV(request.params[2], "parameter 3");
|
||||
blockindex = LookupBlockIndex(blockhash);
|
||||
if (!blockindex) {
|
||||
@ -167,6 +170,11 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
|
||||
in_active_chain = chainActive.Contains(blockindex);
|
||||
}
|
||||
|
||||
bool f_txindex_ready = false;
|
||||
if (g_txindex && !blockindex) {
|
||||
f_txindex_ready = g_txindex->BlockUntilSyncedToCurrentChain();
|
||||
}
|
||||
|
||||
CTransactionRef tx;
|
||||
uint256 hash_block;
|
||||
if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) {
|
||||
@ -176,10 +184,12 @@ UniValue getrawtransaction(const JSONRPCRequest& request)
|
||||
throw JSONRPCError(RPC_MISC_ERROR, "Block not available");
|
||||
}
|
||||
errmsg = "No such transaction found in the provided block";
|
||||
} else if (!g_txindex) {
|
||||
errmsg = "No such mempool transaction. Use -txindex to enable blockchain transaction queries";
|
||||
} else if (!f_txindex_ready) {
|
||||
errmsg = "No such mempool transaction. Blockchain transactions are still in the process of being indexed";
|
||||
} else {
|
||||
errmsg = fTxIndex
|
||||
? "No such mempool or blockchain transaction"
|
||||
: "No such mempool transaction. Use -txindex to enable blockchain transaction queries";
|
||||
errmsg = "No such mempool or blockchain transaction";
|
||||
}
|
||||
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions.");
|
||||
}
|
||||
@ -229,19 +239,18 @@ UniValue gettxoutproof(const JSONRPCRequest& request)
|
||||
oneTxid = hash;
|
||||
}
|
||||
|
||||
LOCK(cs_main);
|
||||
|
||||
CBlockIndex* pblockindex = nullptr;
|
||||
|
||||
uint256 hashBlock;
|
||||
if (!request.params[1].isNull())
|
||||
{
|
||||
if (!request.params[1].isNull()) {
|
||||
LOCK(cs_main);
|
||||
hashBlock = uint256S(request.params[1].get_str());
|
||||
pblockindex = LookupBlockIndex(hashBlock);
|
||||
if (!pblockindex) {
|
||||
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
|
||||
}
|
||||
} else {
|
||||
LOCK(cs_main);
|
||||
|
||||
// Loop through txids and try to find which block they're in. Exit loop once a block is found.
|
||||
for (const auto& tx : setTxids) {
|
||||
const Coin& coin = AccessByTxid(*pcoinsTip, tx);
|
||||
@ -252,6 +261,14 @@ UniValue gettxoutproof(const JSONRPCRequest& request)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Allow txindex to catch up if we need to query it and before we acquire cs_main.
|
||||
if (g_txindex && !pblockindex) {
|
||||
g_txindex->BlockUntilSyncedToCurrentChain();
|
||||
}
|
||||
|
||||
LOCK(cs_main);
|
||||
|
||||
if (pblockindex == nullptr)
|
||||
{
|
||||
CTransactionRef tx;
|
||||
|
||||
66
src/test/txindex_tests.cpp
Normal file
66
src/test/txindex_tests.cpp
Normal file
@ -0,0 +1,66 @@
|
||||
// Copyright (c) 2017-2018 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include <index/txindex.h>
|
||||
#include <script/standard.h>
|
||||
#include <test/test_bitcoin.h>
|
||||
#include <util.h>
|
||||
#include <utiltime.h>
|
||||
#include <validation.h>
|
||||
|
||||
#include <boost/test/unit_test.hpp>
|
||||
|
||||
BOOST_AUTO_TEST_SUITE(txindex_tests)
|
||||
|
||||
BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup)
|
||||
{
|
||||
TxIndex txindex(MakeUnique<TxIndexDB>(1 << 20, true));
|
||||
|
||||
CTransactionRef tx_disk;
|
||||
uint256 block_hash;
|
||||
|
||||
// Transaction should not be found in the index before it is started.
|
||||
for (const auto& txn : m_coinbase_txns) {
|
||||
BOOST_CHECK(!txindex.FindTx(txn->GetHash(), block_hash, tx_disk));
|
||||
}
|
||||
|
||||
// BlockUntilSyncedToCurrentChain should return false before txindex is started.
|
||||
BOOST_CHECK(!txindex.BlockUntilSyncedToCurrentChain());
|
||||
|
||||
txindex.Start();
|
||||
|
||||
// Allow tx index to catch up with the block index.
|
||||
constexpr int64_t timeout_ms = 10 * 1000;
|
||||
int64_t time_start = GetTimeMillis();
|
||||
while (!txindex.BlockUntilSyncedToCurrentChain()) {
|
||||
BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis());
|
||||
MilliSleep(100);
|
||||
}
|
||||
|
||||
// Check that txindex has all txs that were in the chain before it started.
|
||||
for (const auto& txn : m_coinbase_txns) {
|
||||
if (!txindex.FindTx(txn->GetHash(), block_hash, tx_disk)) {
|
||||
BOOST_ERROR("FindTx failed");
|
||||
} else if (tx_disk->GetHash() != txn->GetHash()) {
|
||||
BOOST_ERROR("Read incorrect tx");
|
||||
}
|
||||
}
|
||||
|
||||
// Check that new transactions in new blocks make it into the index.
|
||||
for (int i = 0; i < 10; i++) {
|
||||
CScript coinbase_script_pub_key = GetScriptForDestination(coinbaseKey.GetPubKey().GetID());
|
||||
std::vector<CMutableTransaction> no_txns;
|
||||
const CBlock& block = CreateAndProcessBlock(no_txns, coinbase_script_pub_key);
|
||||
const CTransaction& txn = *block.vtx[0];
|
||||
|
||||
BOOST_CHECK(txindex.BlockUntilSyncedToCurrentChain());
|
||||
if (!txindex.FindTx(txn.GetHash(), block_hash, tx_disk)) {
|
||||
BOOST_ERROR("FindTx failed");
|
||||
} else if (tx_disk->GetHash() != txn.GetHash()) {
|
||||
BOOST_ERROR("Read incorrect tx");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END()
|
||||
@ -5,6 +5,8 @@
|
||||
|
||||
#include <threadinterrupt.h>
|
||||
|
||||
CThreadInterrupt::CThreadInterrupt() : flag(false) {}
|
||||
|
||||
CThreadInterrupt::operator bool() const
|
||||
{
|
||||
return flag.load(std::memory_order_acquire);
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
class CThreadInterrupt
|
||||
{
|
||||
public:
|
||||
CThreadInterrupt();
|
||||
explicit operator bool() const;
|
||||
void operator()();
|
||||
void reset();
|
||||
|
||||
171
src/txdb.cpp
171
src/txdb.cpp
@ -22,6 +22,7 @@ static const char DB_COIN = 'C';
|
||||
static const char DB_COINS = 'c';
|
||||
static const char DB_BLOCK_FILES = 'f';
|
||||
static const char DB_TXINDEX = 't';
|
||||
static const char DB_TXINDEX_BLOCK = 'T';
|
||||
static const char DB_BLOCK_INDEX = 'b';
|
||||
|
||||
static const char DB_BEST_BLOCK = 'B';
|
||||
@ -424,3 +425,173 @@ bool CCoinsViewDB::Upgrade() {
|
||||
LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
|
||||
return !ShutdownRequested();
|
||||
}
|
||||
|
||||
TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) :
|
||||
CDBWrapper(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
|
||||
{}
|
||||
|
||||
bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
|
||||
{
|
||||
return Read(std::make_pair(DB_TXINDEX, txid), pos);
|
||||
}
|
||||
|
||||
bool TxIndexDB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
|
||||
{
|
||||
CDBBatch batch(*this);
|
||||
for (const auto& tuple : v_pos) {
|
||||
batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
|
||||
}
|
||||
return WriteBatch(batch);
|
||||
}
|
||||
|
||||
bool TxIndexDB::ReadBestBlock(CBlockLocator& locator) const
|
||||
{
|
||||
bool success = Read(DB_BEST_BLOCK, locator);
|
||||
if (!success) {
|
||||
locator.SetNull();
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
bool TxIndexDB::WriteBestBlock(const CBlockLocator& locator)
|
||||
{
|
||||
return Write(DB_BEST_BLOCK, locator);
|
||||
}
|
||||
|
||||
/*
|
||||
* Safely persist a transfer of data from the old txindex database to the new one, and compact the
|
||||
* range of keys updated. This is used internally by MigrateData.
|
||||
*/
|
||||
static void WriteTxIndexMigrationBatches(TxIndexDB& newdb, CBlockTreeDB& olddb,
|
||||
CDBBatch& batch_newdb, CDBBatch& batch_olddb,
|
||||
const std::pair<unsigned char, uint256>& begin_key,
|
||||
const std::pair<unsigned char, uint256>& end_key)
|
||||
{
|
||||
// Sync new DB changes to disk before deleting from old DB.
|
||||
newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
|
||||
olddb.WriteBatch(batch_olddb);
|
||||
olddb.CompactRange(begin_key, end_key);
|
||||
|
||||
batch_newdb.Clear();
|
||||
batch_olddb.Clear();
|
||||
}
|
||||
|
||||
bool TxIndexDB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
|
||||
{
|
||||
// The prior implementation of txindex was always in sync with block index
|
||||
// and presence was indicated with a boolean DB flag. If the flag is set,
|
||||
// this means the txindex from a previous version is valid and in sync with
|
||||
// the chain tip. The first step of the migration is to unset the flag and
|
||||
// write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
|
||||
// index entries are copied over in batches to the new database. Finally,
|
||||
// DB_TXINDEX_BLOCK is erased from the old database and the block hash is
|
||||
// written to the new database.
|
||||
//
|
||||
// Unsetting the boolean flag ensures that if the node is downgraded to a
|
||||
// previous version, it will not see a corrupted, partially migrated index
|
||||
// -- it will see that the txindex is disabled. When the node is upgraded
|
||||
// again, the migration will pick up where it left off and sync to the block
|
||||
// with hash DB_TXINDEX_BLOCK.
|
||||
bool f_legacy_flag = false;
|
||||
block_tree_db.ReadFlag("txindex", f_legacy_flag);
|
||||
if (f_legacy_flag) {
|
||||
if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
|
||||
return error("%s: cannot write block indicator", __func__);
|
||||
}
|
||||
if (!block_tree_db.WriteFlag("txindex", false)) {
|
||||
return error("%s: cannot write block index db flag", __func__);
|
||||
}
|
||||
}
|
||||
|
||||
CBlockLocator locator;
|
||||
if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int64_t count = 0;
|
||||
LogPrintf("Upgrading txindex database... [0%%]\n");
|
||||
uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
|
||||
int report_done = 0;
|
||||
const size_t batch_size = 1 << 24; // 16 MiB
|
||||
|
||||
CDBBatch batch_newdb(*this);
|
||||
CDBBatch batch_olddb(block_tree_db);
|
||||
|
||||
std::pair<unsigned char, uint256> key;
|
||||
std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
|
||||
std::pair<unsigned char, uint256> prev_key = begin_key;
|
||||
|
||||
bool interrupted = false;
|
||||
std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
|
||||
for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
|
||||
boost::this_thread::interruption_point();
|
||||
if (ShutdownRequested()) {
|
||||
interrupted = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!cursor->GetKey(key)) {
|
||||
return error("%s: cannot get key from valid cursor", __func__);
|
||||
}
|
||||
if (key.first != DB_TXINDEX) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Log progress every 10%.
|
||||
if (++count % 256 == 0) {
|
||||
// Since txids are uniformly random and traversed in increasing order, the high 16 bits
|
||||
// of the hash can be used to estimate the current progress.
|
||||
const uint256& txid = key.second;
|
||||
uint32_t high_nibble =
|
||||
(static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
|
||||
(static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
|
||||
int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
|
||||
|
||||
uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
|
||||
if (report_done < percentage_done/10) {
|
||||
LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
|
||||
report_done = percentage_done/10;
|
||||
}
|
||||
}
|
||||
|
||||
CDiskTxPos value;
|
||||
if (!cursor->GetValue(value)) {
|
||||
return error("%s: cannot parse txindex record", __func__);
|
||||
}
|
||||
batch_newdb.Write(key, value);
|
||||
batch_olddb.Erase(key);
|
||||
|
||||
if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
|
||||
// NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
|
||||
// because LevelDB iterators are guaranteed to provide a consistent view of the
|
||||
// underlying data, like a lightweight snapshot.
|
||||
WriteTxIndexMigrationBatches(*this, block_tree_db,
|
||||
batch_newdb, batch_olddb,
|
||||
prev_key, key);
|
||||
prev_key = key;
|
||||
}
|
||||
}
|
||||
|
||||
// If these final DB batches complete the migration, write the best block
|
||||
// hash marker to the new database and delete from the old one. This signals
|
||||
// that the former is fully caught up to that point in the blockchain and
|
||||
// that all txindex entries have been removed from the latter.
|
||||
if (!interrupted) {
|
||||
batch_olddb.Erase(DB_TXINDEX_BLOCK);
|
||||
batch_newdb.Write(DB_BEST_BLOCK, locator);
|
||||
}
|
||||
|
||||
WriteTxIndexMigrationBatches(*this, block_tree_db,
|
||||
batch_newdb, batch_olddb,
|
||||
begin_key, key);
|
||||
|
||||
if (interrupted) {
|
||||
LogPrintf("[CANCELLED].\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
uiInterface.ShowProgress("", 100, false);
|
||||
|
||||
LogPrintf("[DONE].\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
38
src/txdb.h
38
src/txdb.h
@ -9,6 +9,7 @@
|
||||
#include <coins.h>
|
||||
#include <dbwrapper.h>
|
||||
#include <chain.h>
|
||||
#include <primitives/block.h>
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
@ -35,7 +36,7 @@ static const int64_t nMaxBlockDBCache = 2;
|
||||
//! Max memory allocated to block tree DB specific cache, if -txindex (MiB)
|
||||
// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
|
||||
// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
|
||||
static const int64_t nMaxBlockDBAndTxIndexCache = 1024;
|
||||
static const int64_t nMaxTxIndexCache = 1024;
|
||||
//! Max memory allocated to coin DB specific cache (MiB)
|
||||
static const int64_t nMaxCoinsDBCache = 8;
|
||||
|
||||
@ -112,9 +113,6 @@ class CBlockTreeDB : public CDBWrapper
|
||||
public:
|
||||
explicit CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
|
||||
|
||||
CBlockTreeDB(const CBlockTreeDB&) = delete;
|
||||
CBlockTreeDB& operator=(const CBlockTreeDB&) = delete;
|
||||
|
||||
bool WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo);
|
||||
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info);
|
||||
bool ReadLastBlockFile(int &nFile);
|
||||
@ -127,4 +125,36 @@ public:
|
||||
bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex);
|
||||
};
|
||||
|
||||
/**
|
||||
* Access to the txindex database (indexes/txindex/)
|
||||
*
|
||||
* The database stores a block locator of the chain the database is synced to
|
||||
* so that the TxIndex can efficiently determine the point it last stopped at.
|
||||
* A locator is used instead of a simple hash of the chain tip because blocks
|
||||
* and block index entries may not be flushed to disk until after this database
|
||||
* is updated.
|
||||
*/
|
||||
class TxIndexDB : public CDBWrapper
|
||||
{
|
||||
public:
|
||||
explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
|
||||
|
||||
/// Read the disk location of the transaction data with the given hash. Returns false if the
|
||||
/// transaction hash is not indexed.
|
||||
bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
|
||||
|
||||
/// Write a batch of transaction positions to the DB.
|
||||
bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
|
||||
|
||||
/// Read block locator of the chain that the txindex is in sync with.
|
||||
bool ReadBestBlock(CBlockLocator& locator) const;
|
||||
|
||||
/// Write block locator of the chain that the txindex is in sync with.
|
||||
bool WriteBestBlock(const CBlockLocator& locator);
|
||||
|
||||
/// Migrate txindex data from the block tree DB, where it may be for older nodes that have not
|
||||
/// been upgraded yet to the new database.
|
||||
bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
|
||||
};
|
||||
|
||||
#endif // BITCOIN_TXDB_H
|
||||
|
||||
@ -16,6 +16,7 @@
|
||||
#include <consensus/validation.h>
|
||||
#include <cuckoocache.h>
|
||||
#include <hash.h>
|
||||
#include <index/txindex.h>
|
||||
#include <init.h>
|
||||
#include <policy/fees.h>
|
||||
#include <policy/policy.h>
|
||||
@ -217,7 +218,6 @@ uint256 g_best_block;
|
||||
int nScriptCheckThreads = 0;
|
||||
std::atomic_bool fImporting(false);
|
||||
std::atomic_bool fReindex(false);
|
||||
bool fTxIndex = false;
|
||||
bool fHavePruned = false;
|
||||
bool fPruneMode = false;
|
||||
bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG;
|
||||
@ -1028,28 +1028,8 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus
|
||||
return true;
|
||||
}
|
||||
|
||||
if (fTxIndex) {
|
||||
CDiskTxPos postx;
|
||||
if (pblocktree->ReadTxIndex(hash, postx)) {
|
||||
CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
|
||||
if (file.IsNull())
|
||||
return error("%s: OpenBlockFile failed", __func__);
|
||||
CBlockHeader header;
|
||||
try {
|
||||
file >> header;
|
||||
fseek(file.Get(), postx.nTxOffset, SEEK_CUR);
|
||||
file >> txOut;
|
||||
} catch (const std::exception& e) {
|
||||
return error("%s: Deserialize or I/O error - %s", __func__, e.what());
|
||||
}
|
||||
hashBlock = header.GetHash();
|
||||
if (txOut->GetHash() != hash)
|
||||
return error("%s: txid mismatch", __func__);
|
||||
return true;
|
||||
}
|
||||
|
||||
// transaction not found in index, nothing more can be done
|
||||
return false;
|
||||
if (g_txindex) {
|
||||
return g_txindex->FindTx(hash, hashBlock, txOut);
|
||||
}
|
||||
|
||||
if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it
|
||||
@ -1668,26 +1648,6 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState&
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool WriteTxIndexDataForBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex)
|
||||
{
|
||||
if (!fTxIndex) return true;
|
||||
|
||||
CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
|
||||
std::vector<std::pair<uint256, CDiskTxPos> > vPos;
|
||||
vPos.reserve(block.vtx.size());
|
||||
for (const CTransactionRef& tx : block.vtx)
|
||||
{
|
||||
vPos.push_back(std::make_pair(tx->GetHash(), pos));
|
||||
pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION);
|
||||
}
|
||||
|
||||
if (!pblocktree->WriteTxIndex(vPos)) {
|
||||
return AbortNode(state, "Failed to write transaction index");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static CCheckQueue<CScriptCheck> scriptcheckqueue(128);
|
||||
|
||||
void ThreadScriptCheck() {
|
||||
@ -2079,9 +2039,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
|
||||
setDirtyBlockIndex.insert(pindex);
|
||||
}
|
||||
|
||||
if (!WriteTxIndexDataForBlock(block, state, pindex))
|
||||
return false;
|
||||
|
||||
assert(pindex->phashBlock);
|
||||
// add this block to the view's block chain
|
||||
view.SetBestBlock(pindex->GetBlockHash());
|
||||
@ -3903,10 +3860,6 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams)
|
||||
pblocktree->ReadReindexing(fReindexing);
|
||||
if(fReindexing) fReindex = true;
|
||||
|
||||
// Check whether we have a transaction index
|
||||
pblocktree->ReadFlag("txindex", fTxIndex);
|
||||
LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled");
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -4300,9 +4253,6 @@ bool LoadBlockIndex(const CChainParams& chainparams)
|
||||
// needs_init.
|
||||
|
||||
LogPrintf("Initializing databases...\n");
|
||||
// Use the provided setting for -txindex in the new database
|
||||
fTxIndex = gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX);
|
||||
pblocktree->WriteFlag("txindex", fTxIndex);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -171,7 +171,6 @@ extern uint256 g_best_block;
|
||||
extern std::atomic_bool fImporting;
|
||||
extern std::atomic_bool fReindex;
|
||||
extern int nScriptCheckThreads;
|
||||
extern bool fTxIndex;
|
||||
extern bool fIsBareMultisigStd;
|
||||
extern bool fRequireStandard;
|
||||
extern bool fCheckBlockIndex;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user