mirrors/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-20 14:05:23 +01:00
Code Issues Projects Releases Wiki Activity

Merge bitcoin/bitcoin#23549: Add scanblocks RPC call (attempt 2)

Browse source 
626b7c8493 fuzz: add scanblocks as safe for fuzzing (James O'Beirne)
94fe5453c7 test: rpc: add scanblocks functional test (Jonas Schnelli)
6ef2566b68 rpc: add scanblocks - scan for relevant blocks with descriptors (Jonas Schnelli)
a4258f6e81 rpc: move-only: consolidate blockchain scan args (James O'Beirne)

Pull request description:

  Revives #20664. All feedback from the previous PR has either been responded to inline or incorporated here.

  ---

  Major changes from Jonas' PR:
  - consolidated arguments for scantxoutset/scanblocks
  - substantial cleanup of the functional test

  Here's the range-diff (`git range-diff master jonasschnelli/2020/12/filterblocks_rpc jamesob/2021-11-scanblocks`): https://gist.github.com/jamesob/aa4a975344209f0316444b8de2ec1d18

  ### Original PR description

  > The `scanblocks` RPC call allows one to get relevant blockhashes from a set of descriptors by scanning all blockfilters in a given range.
  >
  > **Example:**
  >
  > `scanblocks start '["addr(<bitcoin_address>)"]' 661000` (returns relevant blockhashes for `<bitcoin_address>` from blockrange 661000->tip)
  >
  > ## Why is this useful?
  > **Fast wallet rescans**: get the relevant blocks and only rescan those via `rescanblockchain getblockheader(<hash>)[height] getblockheader(<hash>)[height])`. A future PR may add an option to allow to provide an array of blockhashes to `rescanblockchain`.
  >
  > **prune wallet rescans**: (_needs additional changes_): together with a call to fetch blocks from the p2p network if they have been pruned, it would allow to rescan wallets back to the genesis block in pruned mode (relevant #15946).
  >
  > **SPV mode** (_needs additional changes_): it would be possible to build the blockfilterindex from the p2p network (rather then deriving them from the blocks) and thus allow some sort of hybrid-SPV mode with moderate bandwidth consumption (related #9483)

ACKs for top commit:
  furszy:
    diff re-ACK 626b7c8

Tree-SHA512: f84e4dcb851b122b39e9700c58fbc31e899cdcf9b587df9505eaf1f45578cc4253e89ce2a45d1ff21bd213e31ddeedbbcad2c80810f46755b30acc17b07e2873
This commit is contained in:
Andrew Chow 2022-10-13 10:43:08 -04:00
commit bc2b1f0fe2
No known key found for this signature in database
GPG key ID: 17565732E08E5E41
5 changed files with 335 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

258
src/rpc/blockchain.cpp
View file

@ -2019,6 +2019,40 @@ public:
}
};
static const auto scan_action_arg_desc = RPCArg{
"action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n"
"\"start\" for starting a scan\n"
"\"abort\" for aborting the current scan (returns true when abort was successful)\n"
"\"status\" for progress report (in %) of the current scan"
};
static const auto scan_objects_arg_desc = RPCArg{
"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n"
"Every scan object is either a string descriptor or an object:",
{
{"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"},
{"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata",
{
{"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"},
{"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"},
}},
},
RPCArgOptions{.oneline_description="[scanobjects,...]"},
};
static const auto scan_result_abort = RPCResult{
"when action=='abort'", RPCResult::Type::BOOL, "success",
"True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort"
};
static const auto scan_result_status_none = RPCResult{
"when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", ""
};
static const auto scan_result_status_some = RPCResult{
"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "",
{{RPCResult::Type::NUM, "progress", "Approximate percent complete"},}
};
static RPCHelpMan scantxoutset()
{
// scriptPubKey corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S
@ -2038,21 +2072,8 @@ static RPCHelpMan scantxoutset()
"In the latter case, a range needs to be specified by below if different from 1000.\n"
"For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n",
{
{"action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n"
"\"start\" for starting a scan\n"
"\"abort\" for aborting the current scan (returns true when abort was successful)\n"
"\"status\" for progress report (in %) of the current scan"},
{"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n"
"Every scan object is either a string descriptor or an object:",
{
{"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"},
{"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata",
{
{"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"},
{"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"},
}},
},
RPCArgOptions{.oneline_description="[scanobjects,...]"}},
scan_action_arg_desc,
scan_objects_arg_desc,
},
{
RPCResult{"when action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", {
@ -2075,12 +2096,9 @@ static RPCHelpMan scantxoutset()
}},
{RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT},
}},
RPCResult{"when action=='abort'", RPCResult::Type::BOOL, "success", "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort"},
RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "",
{
{RPCResult::Type::NUM, "progress", "Approximate percent complete"},
}},
RPCResult{"when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", ""},
scan_result_abort,
scan_result_status_some,
scan_result_status_none,
},
RPCExamples{
HelpExampleCli("scantxoutset", "start \'[\"" + EXAMPLE_DESCRIPTOR_RAW + "\"]\'") +
@ -2188,6 +2206,203 @@ static RPCHelpMan scantxoutset()
};
}
/** RAII object to prevent concurrency issue when scanning blockfilters */
static std::atomic<int> g_scanfilter_progress;
static std::atomic<int> g_scanfilter_progress_height;
static std::atomic<bool> g_scanfilter_in_progress;
static std::atomic<bool> g_scanfilter_should_abort_scan;
class BlockFiltersScanReserver
{
private:
bool m_could_reserve{false};
public:
explicit BlockFiltersScanReserver() = default;
bool reserve() {
CHECK_NONFATAL(!m_could_reserve);
if (g_scanfilter_in_progress.exchange(true)) {
return false;
}
m_could_reserve = true;
return true;
}
~BlockFiltersScanReserver() {
if (m_could_reserve) {
g_scanfilter_in_progress = false;
}
}
};
static RPCHelpMan scanblocks()
{
return RPCHelpMan{"scanblocks",
"\nReturn relevant blockhashes for given descriptors.\n"
"This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)",
{
scan_action_arg_desc,
scan_objects_arg_desc,
RPCArg{"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "Height to start to scan from"},
RPCArg{"stop_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"chain tip"}, "Height to stop to scan"},
RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC)}, "The type name of the filter"}
},
{
scan_result_status_none,
RPCResult{"When action=='start'", RPCResult::Type::OBJ, "", "", {
{RPCResult::Type::NUM, "from_height", "The height we started the scan from"},
{RPCResult::Type::NUM, "to_height", "The height we ended the scan at"},
{RPCResult::Type::ARR, "relevant_blocks", "", {{RPCResult::Type::STR_HEX, "blockhash", "A relevant blockhash"},}},
},
},
RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", {
{RPCResult::Type::NUM, "progress", "Approximate percent complete"},
{RPCResult::Type::NUM, "current_height", "Height of the block currently being scanned"},
},
},
scan_result_abort,
},
RPCExamples{
HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 300000") +
HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 100 150 basic") +
HelpExampleCli("scanblocks", "status") +
HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 300000") +
HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 100, 150, \"basic\"") +
HelpExampleRpc("scanblocks", "\"status\"")
},
[&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
{
UniValue ret(UniValue::VOBJ);
if (request.params[0].get_str() == "status") {
BlockFiltersScanReserver reserver;
if (reserver.reserve()) {
// no scan in progress
return NullUniValue;
}
ret.pushKV("progress", g_scanfilter_progress.load());
ret.pushKV("current_height", g_scanfilter_progress_height.load());
return ret;
} else if (request.params[0].get_str() == "abort") {
BlockFiltersScanReserver reserver;
if (reserver.reserve()) {
// reserve was possible which means no scan was running
return false;
}
// set the abort flag
g_scanfilter_should_abort_scan = true;
return true;
}
else if (request.params[0].get_str() == "start") {
BlockFiltersScanReserver reserver;
if (!reserver.reserve()) {
throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\"");
}
const std::string filtertype_name{request.params[4].isNull() ? "basic" : request.params[4].get_str()};
BlockFilterType filtertype;
if (!BlockFilterTypeByName(filtertype_name, filtertype)) {
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype");
}
BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
if (!index) {
throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name);
}
NodeContext& node = EnsureAnyNodeContext(request.context);
ChainstateManager& chainman = EnsureChainman(node);
// set the start-height
const CBlockIndex* block = nullptr;
const CBlockIndex* stop_block = nullptr;
{
LOCK(cs_main);
CChain& active_chain = chainman.ActiveChain();
block = active_chain.Genesis();
stop_block = active_chain.Tip();
if (!request.params[2].isNull()) {
block = active_chain[request.params[2].getInt<int>()];
if (!block) {
throw JSONRPCError(RPC_MISC_ERROR, "Invalid start_height");
}
}
if (!request.params[3].isNull()) {
stop_block = active_chain[request.params[3].getInt<int>()];
if (!stop_block || stop_block->nHeight < block->nHeight) {
throw JSONRPCError(RPC_MISC_ERROR, "Invalid stop_height");
}
}
}
CHECK_NONFATAL(block);
// loop through the scan objects, add scripts to the needle_set
GCSFilter::ElementSet needle_set;
for (const UniValue& scanobject : request.params[1].get_array().getValues()) {
FlatSigningProvider provider;
std::vector<CScript> scripts = EvalDescriptorStringOrObject(scanobject, provider);
for (const CScript& script : scripts) {
needle_set.emplace(script.begin(), script.end());
}
}
UniValue blocks(UniValue::VARR);
const int amount_per_chunk = 10000;
const CBlockIndex* start_index = block; // for remembering the start of a blockfilter range
std::vector<BlockFilter> filters;
const CBlockIndex* start_block = block; // for progress reporting
const int total_blocks_to_process = stop_block->nHeight - start_block->nHeight;
g_scanfilter_should_abort_scan = false;
g_scanfilter_progress = 0;
g_scanfilter_progress_height = start_block->nHeight;
while (block) {
node.rpc_interruption_point(); // allow a clean shutdown
if (g_scanfilter_should_abort_scan) {
LogPrintf("scanblocks RPC aborted at height %d.\n", block->nHeight);
break;
}
const CBlockIndex* next = nullptr;
{
LOCK(cs_main);
CChain& active_chain = chainman.ActiveChain();
next = active_chain.Next(block);
if (block == stop_block) next = nullptr;
}
if (start_index->nHeight + amount_per_chunk == block->nHeight || next == nullptr) {
LogPrint(BCLog::RPC, "Fetching blockfilters from height %d to height %d.\n", start_index->nHeight, block->nHeight);
if (index->LookupFilterRange(start_index->nHeight, block, filters)) {
for (const BlockFilter& filter : filters) {
// compare the elements-set with each filter
if (filter.GetFilter().MatchAny(needle_set)) {
blocks.push_back(filter.GetBlockHash().GetHex());
LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex());
}
}
}
start_index = block;
// update progress
int blocks_processed = block->nHeight - start_block->nHeight;
if (total_blocks_to_process > 0) { // avoid division by zero
g_scanfilter_progress = (int)(100.0 / total_blocks_to_process * blocks_processed);
} else {
g_scanfilter_progress = 100;
}
g_scanfilter_progress_height = block->nHeight;
}
block = next;
}
ret.pushKV("from_height", start_block->nHeight);
ret.pushKV("to_height", g_scanfilter_progress_height.load());
ret.pushKV("relevant_blocks", blocks);
}
else {
throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid command");
}
return ret;
},
};
}
static RPCHelpMan getblockfilter()
{
return RPCHelpMan{"getblockfilter",
@ -2423,6 +2638,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t)
{"blockchain", &verifychain},
{"blockchain", &preciousblock},
{"blockchain", &scantxoutset},
{"blockchain", &scanblocks},
{"blockchain", &getblockfilter},
{"hidden", &invalidateblock},
{"hidden", &reconsiderblock},

3
src/rpc/client.cpp
View file

@ -83,6 +83,9 @@ static const CRPCConvertParam vRPCConvertParams[] =
{ "sendmany", 8, "fee_rate"},
{ "sendmany", 9, "verbose" },
{ "deriveaddresses", 1, "range" },
{ "scanblocks", 1, "scanobjects" },
{ "scanblocks", 2, "start_height" },
{ "scanblocks", 3, "stop_height" },
{ "scantxoutset", 1, "scanobjects" },
{ "addmultisigaddress", 0, "nrequired" },
{ "addmultisigaddress", 1, "keys" },

1
src/test/fuzz/rpc.cpp
View file

@ -151,6 +151,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
"preciousblock",
"pruneblockchain",
"reconsiderblock",
"scanblocks",
"scantxoutset",
"sendrawtransaction",
"setmocktime",

93
test/functional/rpc_scanblocks.py Executable file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env python3
# Copyright (c) 2021 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the scanblocks RPC call."""
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import assert_equal, assert_raises_rpc_error
class ScanblocksTest(BitcoinTestFramework):
def set_test_params(self):
self.num_nodes = 2
self.extra_args = [["-blockfilterindex=1"], []]
def skip_test_if_missing_module(self):
self.skip_if_no_wallet()
def run_test(self):
node = self.nodes[0]
# send 1.0, mempool only
addr_1 = node.getnewaddress()
node.sendtoaddress(addr_1, 1.0)
parent_key = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B"
# send 1.0, mempool only
# childkey 5 of `parent_key`
node.sendtoaddress("mkS4HXoTYWRTescLGaUTGbtTTYX5EjJyEE", 1.0)
# mine a block and assure that the mined blockhash is in the filterresult
blockhash = self.generate(node, 1)[0]
height = node.getblockheader(blockhash)['height']
self.wait_until(lambda: all(i["synced"] for i in node.getindexinfo().values()))
out = node.scanblocks("start", [f"addr({addr_1})"])
assert(blockhash in out['relevant_blocks'])
assert_equal(height, out['to_height'])
assert_equal(0, out['from_height'])
# mine another block
blockhash_new = self.generate(node, 1)[0]
height_new = node.getblockheader(blockhash_new)['height']
# make sure the blockhash is not in the filter result if we set the start_height
# to the just mined block (unlikely to hit a false positive)
assert(blockhash not in node.scanblocks(
"start", [f"addr({addr_1})"], height_new)['relevant_blocks'])
# make sure the blockhash is present when using the first mined block as start_height
assert(blockhash in node.scanblocks(
"start", [f"addr({addr_1})"], height)['relevant_blocks'])
# also test the stop height
assert(blockhash in node.scanblocks(
"start", [f"addr({addr_1})"], height, height)['relevant_blocks'])
# use the stop_height to exclude the relevant block
assert(blockhash not in node.scanblocks(
"start", [f"addr({addr_1})"], 0, height - 1)['relevant_blocks'])
# make sure the blockhash is present when using the first mined block as start_height
assert(blockhash in node.scanblocks(
"start", [{"desc": f"pkh({parent_key}/*)", "range": [0, 100]}], height)['relevant_blocks'])
# test node with disabled blockfilterindex
assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic",
self.nodes[1].scanblocks, "start", [f"addr({addr_1})"])
# test unknown filtertype
assert_raises_rpc_error(-5, "Unknown filtertype",
node.scanblocks, "start", [f"addr({addr_1})"], 0, 10, "extended")
# test invalid start_height
assert_raises_rpc_error(-1, "Invalid start_height",
node.scanblocks, "start", [f"addr({addr_1})"], 100000000)
# test invalid stop_height
assert_raises_rpc_error(-1, "Invalid stop_height",
node.scanblocks, "start", [f"addr({addr_1})"], 10, 0)
assert_raises_rpc_error(-1, "Invalid stop_height",
node.scanblocks, "start", [f"addr({addr_1})"], 10, 100000000)
# test accessing the status (must be empty)
assert_equal(node.scanblocks("status"), None)
# test aborting the current scan (there is no, must return false)
assert_equal(node.scanblocks("abort"), False)
# test invalid command
assert_raises_rpc_error(-8, "Invalid command", node.scanblocks, "foobar")
if __name__ == '__main__':
ScanblocksTest().main()

1
test/functional/test_runner.py
View file

@ -316,6 +316,7 @@ BASE_SCRIPTS = [
'rpc_deriveaddresses.py',
'rpc_deriveaddresses.py --usecli',
'p2p_ping.py',
'rpc_scanblocks.py',
'rpc_scantxoutset.py',
'feature_txindex_compatibility.py',
'feature_unsupported_utxo_db.py',