From 220a5a2841172a07d6d7849596316f0e0933e272 Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:08:49 +0200 Subject: [PATCH 1/3] test: hook into PID in tracing tests This makes sure to NOT hook into other bitcoind binaries run in paralell in the test framework. We only want to trace the intended binary. In interface_usdt_utxocache.py: While testing the utxocache flush with pruning, bitcoind is restarted and we need to hook into the new PID again. --- test/functional/interface_usdt_net.py | 2 +- test/functional/interface_usdt_utxocache.py | 15 ++++++++++++--- test/functional/interface_usdt_validation.py | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py index 9522cd8c595..2235da702b2 100755 --- a/test/functional/interface_usdt_net.py +++ b/test/functional/interface_usdt_net.py @@ -109,7 +109,7 @@ class NetTracepointTest(BitcoinTestFramework): self.log.info( "hook into the net:inbound_message and net:outbound_message tracepoints") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="net:inbound_message", fn_name="trace_inbound_message") ctx.enable_probe(probe="net:outbound_message", diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index f48ff9699d8..61587197efb 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -173,7 +173,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16) self.log.info("hooking into the utxocache:uncache tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:uncache", fn_name="trace_utxocache_uncache") bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0) @@ -238,7 +238,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): self.log.info( "hook into the utxocache:add and utxocache:spent tracepoints") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add") ctx.enable_probe(probe="utxocache:spent", fn_name="trace_utxocache_spent") @@ -334,7 +334,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): self.log.info("test the utxocache:flush tracepoint API") self.log.info("hook into the utxocache:flush tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="utxocache:flush", fn_name="trace_utxocache_flush") bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) @@ -373,6 +373,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): self.stop_node(0) bpf.perf_buffer_poll(timeout=200) + bpf.cleanup() self.log.info("check that we don't expect additional flushes") assert_equal(0, len(expected_flushes)) @@ -381,6 +382,14 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) + self.log.info("test the utxocache:flush tracepoint API with pruning") + self.log.info("hook into the utxocache:flush tracepoint") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="utxocache:flush", + fn_name="trace_utxocache_flush") + bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) + bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) + BLOCKS_TO_MINE = 350 self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") self.generate(self.wallet, BLOCKS_TO_MINE) diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py index d11809273bb..fb1a91b9141 100755 --- a/test/functional/interface_usdt_validation.py +++ b/test/functional/interface_usdt_validation.py @@ -94,7 +94,7 @@ class ValidationTracepointTest(BitcoinTestFramework): expected_blocks = list() self.log.info("hook into the validation:block_connected tracepoint") - ctx = USDT(path=str(self.options.bitcoind)) + ctx = USDT(pid=self.nodes[0].process.pid) ctx.enable_probe(probe="validation:block_connected", fn_name="trace_block_connected") bpf = BPF(text=validation_blockconnected_program, From dba6f8234217565957e37516a0ea655f1180d99c Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:17:30 +0200 Subject: [PATCH 2/3] test: adopt USDT utxocache interface tests The USDT interface exposes process internals via the tracepoints. This means, the USDT interface tests somewhat awardly depend on these internals. If internals change, the tests have to adopt to that change. Previously, the USDT interface tests weren't run in the CI so changes could break the USDT interface tests without being noticed (e.g. https://github.com/bitcoin/bitcoin/pull/25486). In fa13375aa3fcb4fd5b9e0d4c69ac31cf66c3209a a 'self.rescan_utxos()' call was added in the 'generate()' function of the test framework. 'rescan_utxos()' causes the UTXO cache to be flushed. In the USDT interface tests for the 'utxocache:flush' trancepoint, 'generate()' is used. As the utxo cache is now flushed more often, the number of flushes the tests expectes need to be adopted. Also, the utxo cache has now a different size when being flushed. The utxocache tracepoint is tested by shutting the node down and pruning blocks, to test the 'for_prune' argument. Changes: - A list 'expected_flushes' is now used which contains 'mode', 'for_prune', and 'size' for each expected flush. - When a flush happens, the expected-flush is removed from the list. This list is checked to be empty (unchanged). - Previously, shutting down caused these two flushes: UTXOCacheFlush(duration=*, mode=ALWAYS, size=104, memory=*, for_prune=False) UTXOCacheFlush(duration=*, mode=ALWAYS, size=0, memory=*, for_prune=False) now it causes these flushes: UTXOCacheFlush(duration=*, mode=ALWAYS, size=2, memory=*, for_prune=False) UTXOCacheFlush(duration=*, mode=ALWAYS, size=0, memory=*, for_prune=False) The 104 UTXOs flushed previously were mainly coinbase UTXOs generated in previous tests and the test setup. These are now already flushed. - In the 'for_prune' test we previously hooked into the tracepoint before mining blocks. This changed to only get notified about the tracepoint being triggered for the prune. Here, the utxo cache is empty already as it has just been flushed in 'generate()'. old: UTXOCacheFlush(duration=*, mode=NONE, size=350, memory=*, for_prune=True) new: UTXOCacheFlush(duration=*, mode=NONE, size=0, memory=*, for_prune=True) --- test/functional/interface_usdt_utxocache.py | 33 +++++++++------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py index 61587197efb..2280de14790 100755 --- a/test/functional/interface_usdt_utxocache.py +++ b/test/functional/interface_usdt_utxocache.py @@ -345,16 +345,17 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): # that the handle_* functions succeeded. EXPECTED_HANDLE_FLUSH_SUCCESS = 3 handle_flush_succeeds = 0 - possible_cache_sizes = set() - expected_flushes = [] + expected_flushes = list() def handle_utxocache_flush(_, data, __): nonlocal handle_flush_succeeds event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents self.log.info(f"handle_utxocache_flush(): {event}") - expected = expected_flushes.pop(0) - assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode]) - possible_cache_sizes.remove(event.size) # fails if size not in set + expected_flushes.remove({ + "mode": FLUSHMODE_NAME[event.mode], + "for_prune": event.for_prune, + "size": event.size + }) # sanity checks only assert(event.memory > 0) assert(event.duration > 0) @@ -363,13 +364,12 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) self.log.info("stop the node to flush the UTXO cache") - UTXOS_IN_CACHE = 104 # might need to be changed if the eariler tests are modified + UTXOS_IN_CACHE = 2 # might need to be changed if the eariler tests are modified # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the # second flush, however it can happen that the order changes. - possible_cache_sizes = {UTXOS_IN_CACHE, 0} - flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False} - expected_flushes.extend([flush_for_shutdown, flush_for_shutdown]) + expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": UTXOS_IN_CACHE}) + expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": 0}) self.stop_node(0) bpf.perf_buffer_poll(timeout=200) @@ -377,11 +377,14 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): self.log.info("check that we don't expect additional flushes") assert_equal(0, len(expected_flushes)) - assert_equal(0, len(possible_cache_sizes)) self.log.info("restart the node with -prune") self.start_node(0, ["-fastprune=1", "-prune=1"]) + BLOCKS_TO_MINE = 350 + self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") + self.generate(self.wallet, BLOCKS_TO_MINE) + self.log.info("test the utxocache:flush tracepoint API with pruning") self.log.info("hook into the utxocache:flush tracepoint") ctx = USDT(pid=self.nodes[0].process.pid) @@ -390,15 +393,8 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0) bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush) - BLOCKS_TO_MINE = 350 - self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune") - self.generate(self.wallet, BLOCKS_TO_MINE) - # we added BLOCKS_TO_MINE coinbase UTXOs to the cache - possible_cache_sizes = {BLOCKS_TO_MINE} - expected_flushes.append( - {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE}) - self.log.info(f"prune blockchain to trigger a flush for pruning") + expected_flushes.append({"mode": "NONE", "for_prune": True, "size": 0}) self.nodes[0].pruneblockchain(315) bpf.perf_buffer_poll(timeout=500) @@ -407,7 +403,6 @@ class UTXOCacheTracepointTest(BitcoinTestFramework): self.log.info( f"check that we don't expect additional flushes and that the handle_* function succeeded") assert_equal(0, len(expected_flushes)) - assert_equal(0, len(possible_cache_sizes)) assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds) From cc7335edc87c6ef34429b4df94f53973db520aac Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Fri, 1 Jul 2022 13:49:55 +0200 Subject: [PATCH 3/3] ci: run USDT interface test in a VM Our CI tasks are run by CirrusCI in Docker containers in a Google Compute Engine based Kubernetes environment. These containers have limited capabilities - especially CAP_SYS_ADMIN is missing. See https://github.com/bitcoin/bitcoin/issues/23296#issuecomment-1024920845 We need elevated privileges to hook into the USDT tracepoints. We use a CirrusCI "compute_engine_instance" (a VM, not a container) where we have the required privileges. The ubunut-mininmal-2204-lts was choosen with debian-11 being an alternative. Both pack an outdated 'bpfcc-tools' package (v0.18.0) from 2020. This version prints warnings to stderr during BPF bytecode compilation, which causes our functional test runner to fail. This is fixed in newer verison. Until debian-12 or a newer Ubuntu release is avaliable as image in GCE (https://cloud.google.com/compute/docs/images/os-details), we use a third-party and untrusted PPA that releases up-to-date versions of the package. The official iovisor (authors of BCC) PPA is outdated too. An alternative would be to compile BCC from source in the CI. Co-authored-by: MacroFake --- .cirrus.yml | 14 +++++++++++--- ci/test/00_setup_env_native_asan.sh | 11 ++++++++--- ci/test/00_setup_env_native_tidy.sh | 2 +- ci/test/04_install.sh | 8 ++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index fcf1200d35f..0a8d3d6e6f1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -259,12 +259,20 @@ task: MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan task: - name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' + name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]' << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:jammy + # We can't use a 'container' for the USDT interface tests as the CirrusCI + # containers don't have privileges to hook into bitcoind. CirrusCI uses + # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/ + # Images can be found here: https://cloud.google.com/compute/docs/images/os-details + compute_engine_instance: + image_project: ubuntu-os-cloud + image: family/ubuntu-2204-lts # when upgrading, check if we can drop "ADD_UNTRUSTED_BPFCC_PPA" + cpu: 4 + memory: 12G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + HOME: /root/ # Only needed for compute_engine_instance FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" MAKEJOBS: "-j4" # Avoid excessive memory use diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 69883e3609f..4f1792a9f08 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -6,9 +6,14 @@ export LC_ALL=C.UTF-8 +# We install an up-to-date 'bpfcc-tools' package from an untrusted PPA. +# This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages. +# See the if-then in ci/test/04_install.sh too. +export ADD_UNTRUSTED_BPFCC_PPA=true + export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:22.04 +export PACKAGES="systemtap-sdt-dev bpfcc-tools clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" +export DOCKER_NAME_TAG=ubuntu:22.04 # May not run in docker unless --enable-usdt is dropped export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-c++20 --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index e4d34684734..11a12e336a2 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -15,5 +15,5 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=false export RUN_TIDY=true export GOAL="install" -export BITCOIN_CONFIG="CC=clang CXX=clang++ --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" +export BITCOIN_CONFIG="CC=clang CXX=clang++ --enable-c++20 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" export CCACHE_SIZE=200M diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 453a34ca78e..b706fb09062 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -68,6 +68,14 @@ if [[ $DOCKER_NAME_TAG == *centos* ]]; then ${CI_RETRY_EXE} CI_EXEC dnf -y install epel-release ${CI_RETRY_EXE} CI_EXEC dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then + if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then + # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. + # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated + # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools + # package. + # TODO: drop this once we can use newer images in GCE + CI_EXEC add-apt-repository ppa:hadret/bpfcc + fi ${CI_RETRY_EXE} CI_EXEC apt-get update ${CI_RETRY_EXE} CI_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" if [ -n "$PIP_PACKAGES" ]; then