From c8176f758b5991c3797c32ee519d32c14b444991 Mon Sep 17 00:00:00 2001 From: tdb3 <106488469+tdb3@users.noreply.github.com> Date: Tue, 20 Aug 2024 19:28:14 -0400 Subject: [PATCH 1/6] test: add blocks_key_path Adds a convenience function to TestNode to provide the path to the blocks xor key. Updates util and feature_blocksxor to use it. --- test/functional/feature_blocksxor.py | 2 +- test/functional/test_framework/test_node.py | 4 ++++ test/functional/test_framework/util.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 88e0244cd42..e9d89ede8f5 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -54,7 +54,7 @@ class BlocksXORTest(BitcoinTestFramework): match=ErrorMatch.PARTIAL_REGEX) self.log.info("Delete XOR key, restart node with '-blocksxor=0', check blk*.dat/rev*.dat file integrity") - os.remove(node.blocks_path / 'xor.dat') + os.remove(node.blocks_key_path) self.start_node(0, extra_args=['-blocksxor=0']) # checklevel=2 -> verify block validity + undo data # nblocks=0 -> verify all blocks diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index b73566b0e95..9e037345f36 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -465,6 +465,10 @@ class TestNode(): def blocks_path(self) -> Path: return self.chain_path / "blocks" + @property + def blocks_key_path(self) -> Path: + return self.blocks_path / "xor.dat" + @property def wallets_path(self) -> Path: return self.chain_path / "wallets" diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index c3bc861cf6e..2372683c042 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -516,7 +516,7 @@ def check_node_connections(*, node, num_in, num_out): def read_xor_key(*, node): - with open(node.blocks_path / "xor.dat", "rb") as xor_f: + with open(node.blocks_key_path, "rb") as xor_f: NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size() return xor_f.read(NUM_XOR_BYTES) From d43948c3ef610c383176bf9b389697973bd0ad64 Mon Sep 17 00:00:00 2001 From: tdb3 <106488469+tdb3@users.noreply.github.com> Date: Sun, 25 Aug 2024 08:41:21 -0400 Subject: [PATCH 2/6] refactor: use unlink rather than os.remove --- test/functional/feature_blocksxor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index e9d89ede8f5..16597e6d01e 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -3,7 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test support for XORed block data and undo files (`-blocksxor` option).""" -import os from test_framework.test_framework import BitcoinTestFramework from test_framework.test_node import ErrorMatch @@ -54,7 +53,7 @@ class BlocksXORTest(BitcoinTestFramework): match=ErrorMatch.PARTIAL_REGEX) self.log.info("Delete XOR key, restart node with '-blocksxor=0', check blk*.dat/rev*.dat file integrity") - os.remove(node.blocks_key_path) + node.blocks_key_path.unlink() self.start_node(0, extra_args=['-blocksxor=0']) # checklevel=2 -> verify block validity + undo data # nblocks=0 -> verify all blocks From d8399584dd59b3954a0bea393b2de350a732055e Mon Sep 17 00:00:00 2001 From: tdb3 <106488469+tdb3@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:03:25 -0400 Subject: [PATCH 3/6] refactor: move read_xor_key() to TestNode --- test/functional/feature_blocksxor.py | 3 +-- test/functional/feature_reindex.py | 3 +-- test/functional/test_framework/test_node.py | 5 +++++ test/functional/test_framework/util.py | 6 ------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 16597e6d01e..8ea16db7bbf 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -9,7 +9,6 @@ from test_framework.test_node import ErrorMatch from test_framework.util import ( assert_equal, assert_greater_than, - read_xor_key, util_xor, ) from test_framework.wallet import MiniWallet @@ -39,7 +38,7 @@ class BlocksXORTest(BitcoinTestFramework): self.log.info("Shut down node and un-XOR block/undo files manually") self.stop_node(0) - xor_key = read_xor_key(node=node) + xor_key = node.read_xor_key() for data_file in sorted(block_files + undo_files): self.log.debug(f"Rewriting file {data_file}...") with open(data_file, 'rb+') as f: diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index 2961a2d3564..1ebfe82da54 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -14,7 +14,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import MAGIC_BYTES from test_framework.util import ( assert_equal, - read_xor_key, util_xor, ) @@ -43,7 +42,7 @@ class ReindexTest(BitcoinTestFramework): # we're generating them rather than getting them from peers), so to # test out-of-order handling, swap blocks 1 and 2 on disk. blk0 = self.nodes[0].blocks_path / "blk00000.dat" - xor_dat = read_xor_key(node=self.nodes[0]) + xor_dat = self.nodes[0].read_xor_key() with open(blk0, 'r+b') as bf: # Read at least the first few blocks (including genesis) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 9e037345f36..ce34ef6f5f4 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -469,6 +469,11 @@ class TestNode(): def blocks_key_path(self) -> Path: return self.blocks_path / "xor.dat" + def read_xor_key(self) -> bytes: + with open(self.blocks_key_path, "rb") as xor_f: + NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size() + return xor_f.read(NUM_XOR_BYTES) + @property def wallets_path(self) -> Path: return self.chain_path / "wallets" diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 2372683c042..00fe5b08e4c 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -515,12 +515,6 @@ def check_node_connections(*, node, num_in, num_out): assert_equal(info["connections_out"], num_out) -def read_xor_key(*, node): - with open(node.blocks_key_path, "rb") as xor_f: - NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size() - return xor_f.read(NUM_XOR_BYTES) - - # Transaction/Block functions ############################# From 1ad999b9da39b60e16c51f9813f4fd39b7bdc2b9 Mon Sep 17 00:00:00 2001 From: tdb3 <106488469+tdb3@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:12:00 -0400 Subject: [PATCH 4/6] refactor: lift NUM_XOR_BYTES --- test/functional/test_framework/test_node.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ce34ef6f5f4..4ac7db2030e 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -43,6 +43,9 @@ from .util import ( ) BITCOIND_PROC_WAIT_TIMEOUT = 60 +# The size of the blocks xor key +# from InitBlocksdirXorKey::xor_key.size() +NUM_XOR_BYTES = 8 class FailedToStartError(Exception): @@ -471,7 +474,6 @@ class TestNode(): def read_xor_key(self) -> bytes: with open(self.blocks_key_path, "rb") as xor_f: - NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size() return xor_f.read(NUM_XOR_BYTES) @property From d1610962bf1ff14df45c57cc1d2e075f71fcd19a Mon Sep 17 00:00:00 2001 From: tdb3 <106488469+tdb3@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:14:23 -0400 Subject: [PATCH 5/6] test: add null block xor key --- test/functional/test_framework/test_node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 4ac7db2030e..60ca9269a5b 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -46,6 +46,8 @@ BITCOIND_PROC_WAIT_TIMEOUT = 60 # The size of the blocks xor key # from InitBlocksdirXorKey::xor_key.size() NUM_XOR_BYTES = 8 +# The null blocks key (all 0s) +NULL_BLK_XOR_KEY = bytes([0] * NUM_XOR_BYTES) class FailedToStartError(Exception): From e1d5dd732d5dc641faf1dde316275c84b6bb224b Mon Sep 17 00:00:00 2001 From: tdb3 <106488469+tdb3@users.noreply.github.com> Date: Tue, 20 Aug 2024 20:19:46 -0400 Subject: [PATCH 6/6] test: check xor.dat recreated when missing --- test/functional/feature_blocksxor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/functional/feature_blocksxor.py b/test/functional/feature_blocksxor.py index 8ea16db7bbf..7698a66ec40 100755 --- a/test/functional/feature_blocksxor.py +++ b/test/functional/feature_blocksxor.py @@ -5,7 +5,10 @@ """Test support for XORed block data and undo files (`-blocksxor` option).""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.test_node import ErrorMatch +from test_framework.test_node import ( + ErrorMatch, + NULL_BLK_XOR_KEY, +) from test_framework.util import ( assert_equal, assert_greater_than, @@ -57,6 +60,8 @@ class BlocksXORTest(BitcoinTestFramework): # checklevel=2 -> verify block validity + undo data # nblocks=0 -> verify all blocks node.verifychain(checklevel=2, nblocks=0) + self.log.info("Check that blocks XOR key is recreated") + assert_equal(node.read_xor_key(), NULL_BLK_XOR_KEY) if __name__ == '__main__':