Merge #17091: tests: Add test for loadblock option and linearize scripts

89339d1460 tests: Add test for loadblock option (Fabian Jahr)

Pull request description:

  Fixes #17019

  Was initially part of #17044 but as the test got larger it made sense to split it into its own commit as suggested in #17019 .

  This is testing the `-loadblock` option by using the scripts in `contrib/linearize` to generate a `bootstrap.dat` file and starting a disconnected node with it. So it is also testing the linearize scripts which were untested before and needed to be made available for the CI environment, hence they are added to `DIST_CONTRIB` in `Makefile.am`.

ACKs for top commit:
  laanwj:
    ACK 89339d1460

Tree-SHA512: aede0cd6e8b21194973f3633bc07fa2672d66a6f85dfe6a57cee2bb269a65d19ea49d5f9ed7914a173b3847c76e70257aa865f44bde170c1999d9655b4862d1c

Makefile.am

@@ -52,7 +52,10 @@ DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \
 	       $(top_srcdir)/contrib/bitcoind.bash-completion \
 	       $(top_srcdir)/contrib/debian/copyright \
 	       $(top_srcdir)/contrib/init \
-	       $(top_srcdir)/contrib/install_db4.sh
+	       $(top_srcdir)/contrib/install_db4.sh \
+	       $(top_srcdir)/contrib/linearize/linearize-data.py \
+	       $(top_srcdir)/contrib/linearize/linearize-hashes.py
+
 DIST_SHARE = \
   $(top_srcdir)/share/genbuild.sh \
   $(top_srcdir)/share/rpcauth

test/functional/feature_loadblock.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# Copyright (c) 2017-2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test loadblock option
+
+Test the option to start a node with the option loadblock which loads
+a serialized blockchain from a file (usually called bootstrap.dat).
+To generate that file this test uses the helper scripts available
+in contrib/linearize.
+"""
+
+import os
+import subprocess
+import sys
+import tempfile
+import urllib
+
+from test_framework.test_framework import (
+    BitcoinTestFramework,
+)
+from test_framework.util import assert_equal, wait_until
+
+
+class LoadblockTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+
+    def run_test(self):
+        self.nodes[1].setnetworkactive(state=False)
+        self.nodes[0].generate(100)
+
+        # Parsing the url of our node to get settings for config file
+        data_dir = self.nodes[0].datadir
+        node_url = urllib.parse.urlparse(self.nodes[0].url)
+        cfg_file = os.path.join(data_dir, "linearize.cfg")
+        bootstrap_file = os.path.join(self.options.tmpdir, "bootstrap.dat")
+        genesis_block = self.nodes[0].getblockhash(0)
+        blocks_dir = os.path.join(data_dir, "regtest", "blocks")
+        hash_list = tempfile.NamedTemporaryFile(dir=data_dir,
+                                                mode='w',
+                                                delete=False,
+                                                encoding="utf-8")
+
+        self.log.info("Create linearization config file")
+        with open(cfg_file, "a", encoding="utf-8") as cfg:
+            cfg.write("datadir={}\n".format(data_dir))
+            cfg.write("rpcuser={}\n".format(node_url.username))
+            cfg.write("rpcpassword={}\n".format(node_url.password))
+            cfg.write("port={}\n".format(node_url.port))
+            cfg.write("host={}\n".format(node_url.hostname))
+            cfg.write("output_file={}\n".format(bootstrap_file))
+            cfg.write("max_height=100\n")
+            cfg.write("netmagic=fabfb5da\n")
+            cfg.write("input={}\n".format(blocks_dir))
+            cfg.write("genesis={}\n".format(genesis_block))
+            cfg.write("hashlist={}\n".format(hash_list.name))
+
+        base_dir = self.config["environment"]["SRCDIR"]
+        linearize_dir = os.path.join(base_dir, "contrib", "linearize")
+
+        self.log.info("Run linearization of block hashes")
+        linearize_hashes_file = os.path.join(linearize_dir, "linearize-hashes.py")
+        subprocess.run([sys.executable, linearize_hashes_file, cfg_file],
+                       stdout=hash_list,
+                       check=True)
+
+        self.log.info("Run linearization of block data")
+        linearize_data_file = os.path.join(linearize_dir, "linearize-data.py")
+        subprocess.run([sys.executable, linearize_data_file, cfg_file],
+                       check=True)
+
+        self.log.info("Restart second, unsynced node with bootstrap file")
+        self.stop_node(1)
+        self.start_node(1, ["-loadblock=" + bootstrap_file])
+        wait_until(lambda: self.nodes[1].getblockcount() == 100)
+
+        assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 100)
+        assert_equal(self.nodes[0].getbestblockhash(), self.nodes[1].getbestblockhash())
+
+
+if __name__ == '__main__':
+    LoadblockTest().main()

test/functional/test_runner.py

@@ -197,6 +197,7 @@ BASE_SCRIPTS = [
     'feature_uacomment.py',
     'wallet_coinbase_category.py',
     'feature_filelock.py',
+    'feature_loadblock.py',
     'p2p_dos_header_tree.py',
     'p2p_unrequested_blocks.py',
     'feature_includeconf.py',