Merge pull request #3020 from tnull/2024-04-drop-electrsd-autodownload-for-good

Drop `electrsd`'s auto-download feature for good

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Rust ${{ matrix.toolchain }} toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }}
@@ -32,14 +32,38 @@ jobs:
         run: |
           rustup target add thumbv7m-none-eabi
           sudo apt-get -y install gcc-arm-none-eabi
-      - name: shellcheck the CI script
+      - name: shellcheck the CI and `contrib` scripts
         if: "matrix.platform == 'ubuntu-latest'"
         run: |
           sudo apt-get -y install shellcheck
-          shellcheck ci/ci-tests.sh
+          shellcheck ci/*.sh -aP ci
+          shellcheck contrib/*.sh -aP contrib
       - name: Set RUSTFLAGS to deny warnings
         if: "matrix.toolchain == '1.63.0'"
         run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
+      - name: Enable caching for bitcoind
+        id: cache-bitcoind
+        uses: actions/cache@v4
+        with:
+          path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
+          key: bitcoind-${{ runner.os }}-${{ runner.arch }}
+      - name: Enable caching for electrs
+        id: cache-electrs
+        uses: actions/cache@v4
+        with:
+          path: bin/electrs-${{ runner.os }}-${{ runner.arch }}
+          key: electrs-${{ runner.os }}-${{ runner.arch }}
+      - name: Download bitcoind/electrs
+        if: "matrix.platform != 'windows-latest' && (steps.cache-bitcoind.outputs.cache-hit != 'true' || steps.cache-electrs.outputs.cache-hit != 'true')"
+        run: |
+            source ./contrib/download_bitcoind_electrs.sh
+            mkdir bin
+            mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }}
+            mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }}
+      - name: Set bitcoind/electrs environment variables
+        run: |
+            echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
+            echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV"
       - name: Run CI script
         shell: bash # Default on Winblows is powershell
         run: CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh
@@ -50,7 +74,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Rust stable toolchain
@@ -72,14 +96,14 @@ jobs:
       TOOLCHAIN: stable
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
           rustup override set ${{ env.TOOLCHAIN }}
       - name: Cache routing graph snapshot
         id: cache-graph
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: lightning/net_graph-2023-12-10.bin
           key: ldk-net_graph-v0.0.118-2023-12-10.bin
@@ -96,7 +120,7 @@ jobs:
           EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: e94b38ef4b3ce683893bf6a3ee28d60cb37c73b059403ff77b7e7458157968c2
       - name: Cache scorer snapshot
         id: cache-scorer
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: lightning/scorer-2023-12-10.bin
           key: ldk-scorer-v0.0.118-2023-12-10.bin
@@ -142,7 +166,7 @@ jobs:
       TOOLCHAIN: stable
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
@@ -165,7 +189,7 @@ jobs:
       TOOLCHAIN: stable
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
@@ -194,7 +218,7 @@ jobs:
       TOOLCHAIN: 1.63
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
@@ -217,7 +241,7 @@ jobs:
       TOOLCHAIN: stable
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
@@ -235,7 +259,7 @@ jobs:
       TOOLCHAIN: 1.63.0
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
@@ -250,7 +274,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.ref_name != 'main' # `main` has no diff with itself
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Relative diff

ci/check-compiles.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 set -x
-echo Testing $(git log -1 --oneline)
+echo "Testing $(git log -1 --oneline)"
 cargo check
 cargo doc
 cargo doc --document-private-items

ci/check-each-commit.sh

@@ -12,4 +12,4 @@ if [ "$(git log --pretty="%H %D" | grep "^[0-9a-f]*.* $1")" = "" ]; then
 	echo "It seems like the current checked-out commit is not based on $1"
 	exit 1
 fi
-git rebase --exec ci/check-compiles.sh $1
+git rebase --exec ci/check-compiles.sh "$1"

ci/ci-tests.sh

@@ -14,50 +14,6 @@ function PIN_RELEASE_DEPS {
 	return 0 # Don't fail the script if our rustc is higher than the last check
 }
 
-# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind`
-# binaries. Here, we download the binaries, validate them, and export their
-# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
-# `electrsd`/`bitcoind` crates in our tests.
-function DOWNLOAD_ELECTRS_AND_BITCOIND {
-	ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
-	ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
-	BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
-	BITCOIND_VERSION="25.1"
-	if [[ "$HOST_PLATFORM" == *linux* ]]; then
-		ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
-		ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
-		BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
-		BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
-	elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
-		ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
-		ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
-		BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
-		BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
-	else
-		echo -e "\n\nUnsupported platform. Exiting.."
-		exit 1
-	fi
-
-	DL_TMP_DIR=$(mktemp -d)
-	trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
-
-	pushd "$DL_TMP_DIR"
-	ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
-	curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
-	echo "$ELECTRS_DL_HASH  $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
-	unzip "$ELECTRS_DL_FILE_NAME"
-	export ELECTRS_EXE="$DL_TMP_DIR"/electrs
-	chmod +x "$ELECTRS_EXE"
-
-	BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
-	curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
-	echo "$BITCOIND_DL_HASH  $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
-	tar xzf "$BITCOIND_DL_FILE_NAME"
-	export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
-	chmod +x "$BITCOIND_EXE"
-	popd
-}
-
 PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
 
 # Starting with version 1.10.0, the `regex` crate has an MSRV of rustc 1.65.0.
@@ -71,9 +27,6 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace
 
 export RUST_BACKTRACE=1
 
-# Build `lightning-transaction-sync` in no_download mode.
-export RUSTFLAGS="$RUSTFLAGS --cfg no_download"
-
 echo -e "\n\nBuilding and testing all workspace crates..."
 cargo test --verbose --color always
 cargo check --verbose --color always
@@ -91,11 +44,12 @@ cargo check --verbose --color always --features rpc-client,rest-client,tokio
 popd
 
 if [[ "$HOST_PLATFORM" != *windows* ]]; then
+	if [ -z "$BITCOIND_EXE" ] || [ -z "$ELECTRS_EXE" ]; then
+		echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset."
+	else
 		echo -e "\n\nBuilding and testing Transaction Sync Clients with features"
 		pushd lightning-transaction-sync
 
-	DOWNLOAD_ELECTRS_AND_BITCOIND
-
 		cargo test --verbose --color always --features esplora-blocking
 		cargo check --verbose --color always --features esplora-blocking
 		cargo test --verbose --color always --features esplora-async
@@ -106,6 +60,7 @@ if [[ "$HOST_PLATFORM" != *windows* ]]; then
 		cargo check --verbose --color always --features electrum
 		popd
 	fi
+fi
 
 echo -e "\n\nTest futures builds"
 pushd lightning-background-processor

ci/rustfmt.sh

@@ -14,8 +14,8 @@ VERS=""
 
 # Run fmt
 TMP_FILE=$(mktemp)
-find . -name '*.rs' -type f |sort >$TMP_FILE
+find . -name '*.rs' -type f |sort >"$TMP_FILE"
-for file in $(comm -23 $TMP_FILE rustfmt_excluded_files); do
+for file in $(comm -23 "$TMP_FILE" rustfmt_excluded_files); do
 	echo "Checking formatting of $file"
-	rustfmt $VERS --edition 2021 --check $file
+	rustfmt $VERS --edition 2021 --check "$file"
 done

contrib/download_bitcoind_electrs.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -eox pipefail
+
+# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind`
+# binaries. Here, we download the binaries, validate them, and export their
+# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the
+# `electrsd`/`bitcoind` crates in our tests.
+
+HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')"
+ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases"
+ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254"
+BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/"
+BITCOIND_VERSION="25.1"
+if [[ "$HOST_PLATFORM" == *linux* ]]; then
+	ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip
+	ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1"
+	BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz
+	BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b"
+elif [[ "$HOST_PLATFORM" == *darwin* ]]; then
+	ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip
+	ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd"
+	BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz
+	BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1"
+else
+	printf "\n\n"
+	echo "Unsupported platform: $HOST_PLATFORM Exiting.."
+	exit 1
+fi
+
+DL_TMP_DIR=$(mktemp -d)
+trap 'rm -rf -- "$DL_TMP_DIR"' EXIT
+
+pushd "$DL_TMP_DIR"
+ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME"
+curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL"
+echo "$ELECTRS_DL_HASH  $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c
+unzip "$ELECTRS_DL_FILE_NAME"
+export ELECTRS_EXE="$DL_TMP_DIR"/electrs
+chmod +x "$ELECTRS_EXE"
+
+BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME"
+curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL"
+echo "$BITCOIND_DL_HASH  $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c
+tar xzf "$BITCOIND_DL_FILE_NAME"
+export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind
+chmod +x "$BITCOIND_EXE"
+popd

contrib/run-rustfmt.sh

@@ -14,8 +14,8 @@ VERS=""
 
 # Run fmt
 TMP_FILE=$(mktemp)
-find . -name '*.rs' -type f |sort >$TMP_FILE
+find . -name '*.rs' -type f |sort >"$TMP_FILE"
-for file in $(comm -23 $TMP_FILE rustfmt_excluded_files); do
+for file in $(comm -23 "$TMP_FILE" rustfmt_excluded_files); do
 	echo "Formatting $file..."
-	rustfmt $VERS --edition 2021 $file
+	rustfmt $VERS --edition 2021 "$file"
 done

lightning-transaction-sync/Cargo.toml

@@ -34,10 +34,7 @@ electrum-client = { version = "0.19.0", optional = true }
 lightning = { version = "0.0.123-beta", path = "../lightning", default-features = false, features = ["std", "_test_utils"] }
 tokio = { version = "1.35.0", features = ["full"] }
 
-[target.'cfg(all(not(target_os = "windows"), not(no_download)))'.dev-dependencies]
+[target.'cfg(not(target_os = "windows"))'.dev-dependencies]
-electrsd = { version = "0.27.3", default-features = false, features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] }
-
-[target.'cfg(all(not(target_os = "windows"), no_download))'.dev-dependencies]
 electrsd = { version = "0.27.3", default-features = false, features = ["legacy"] }
 
 [lints]