From 42249d5ffab9c6ddb5a30340d374b13e97b8089b Mon Sep 17 00:00:00 2001 From: Bastien Teinturier <31281497+t-bast@users.noreply.github.com> Date: Wed, 16 Aug 2023 17:10:03 +0200 Subject: [PATCH] Update to bitcoind 24.1 (#2711) This lets us use the new `gettxspendingprevout` instead of fetching the whole mempool when looking for txs spending one of our channels. A new feature was added to bitcoind 24.1+ that tries to make the change output indistinguishable from the payment output. This is a great for privacy, but it adds randomness to coin selection and uses a non-minimal set of inputs sometimes. We work around this in tests by updating the amount of the output we want bitcoind to use to make sure it's sufficient to pay for both the channel funding and the change output. This shouldn't be too much of an issue for normal operation, where we'll sometimes use two inputs instead of one, which costs more fees, but increases privacy. See https://github.com/bitcoin/bitcoin/pull/24494 for details. --- README.md | 2 +- eclair-core/pom.xml | 18 ++++----- .../blockchain/bitcoind/ZmqWatcher.scala | 39 +++++++++---------- .../bitcoind/rpc/BitcoinCoreClient.scala | 17 +++++++- .../test/resources/integration/bitcoin.conf | 3 +- .../bitcoind/BitcoinCoreClientSpec.scala | 5 +++ .../blockchain/bitcoind/BitcoindService.scala | 2 +- .../channel/InteractiveTxBuilderSpec.scala | 20 +++++----- 8 files changed, 62 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 6281d9af8..f90dd7b6b 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ This means that instead of re-implementing them, Eclair benefits from the verifi * Eclair needs a _synchronized_, _segwit-ready_, **_zeromq-enabled_**, _wallet-enabled_, _non-pruning_, _tx-indexing_ [Bitcoin Core](https://github.com/bitcoin/bitcoin) node. * You must configure your Bitcoin node to use `bech32` or `bech32m` (segwit) addresses. If your wallet has "non-segwit UTXOs" (outputs that are neither `p2sh-segwit`, `bech32` or `bech32m`), you must send them to a `bech32` or `bech32m` address before running Eclair. -* Eclair requires Bitcoin Core 23.2 or higher. If you are upgrading an existing wallet, you may need to create a new address and send all your funds to that address. +* Eclair requires Bitcoin Core 24.1 or higher. If you are upgrading an existing wallet, you may need to create a new address and send all your funds to that address. Run bitcoind with the following minimal `bitcoin.conf`: diff --git a/eclair-core/pom.xml b/eclair-core/pom.xml index d3a29eb1a..fc3177ad2 100644 --- a/eclair-core/pom.xml +++ b/eclair-core/pom.xml @@ -88,9 +88,9 @@ true - https://bitcoincore.org/bin/bitcoin-core-23.2/bitcoin-23.2-x86_64-linux-gnu.tar.gz - 1ec6ca817c679f3b4b6daa8021e401c7 - 089476b853d8e33a52f67ffc46197dc49aa8a656 + https://bitcoincore.org/bin/bitcoin-core-24.1/bitcoin-24.1-x86_64-linux-gnu.tar.gz + 35a2faf826a9d866aa6821832e39231e + a006ef05514b95cf4d78b5ec844e6cf78fabc196 @@ -101,9 +101,9 @@ - https://bitcoincore.org/bin/bitcoin-core-23.2/bitcoin-23.2-x86_64-apple-darwin.tar.gz - 406feabaad970a70ba10991577536970 - 37165f9ccc23a8bea6a1029cd1186090c01b737c + https://bitcoincore.org/bin/bitcoin-core-24.1/bitcoin-24.1-x86_64-apple-darwin.tar.gz + eba2d59fc9b81c0f6a9ccf77639b8b57 + b75f30dfa9095c733a9402e243e18174de4522d6 @@ -114,9 +114,9 @@ - https://bitcoincore.org/bin/bitcoin-core-23.2/bitcoin-23.2-win64.zip - ba46d7646039bfcaa13137d953ff58bf - 684c669b1c929485422c48f3db6e8bc8304e55f7 + https://bitcoincore.org/bin/bitcoin-core-24.1/bitcoin-24.1-win64.zip + 3a6cff40522e392f4ab1d8aef1274a9d + 588a60fe5d0b4d8fd09fae6bf366c3f0fc9336b8 diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala index 5455a06a7..558877411 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/ZmqWatcher.scala @@ -380,29 +380,28 @@ private class ZmqWatcher(nodeParams: NodeParams, blockHeight: AtomicLong, client // the output has been spent, let's find the spending tx // if we know some potential spending txs, we try to fetch them directly Future.sequence(w.hints.map(txid => client.getTransaction(txid).map(Some(_)).recover { case _ => None })) - .map(_ - .flatten // filter out errors - .find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match { - case Some(spendingTx) => - // there can be only one spending tx for an utxo - log.info(s"${w.txId}:${w.outputIndex} has already been spent by a tx provided in hints: txid=${spendingTx.txid}") - context.self ! ProcessNewTransaction(spendingTx) - case None => - // no luck, we have to do it the hard way... - log.info(s"${w.txId}:${w.outputIndex} has already been spent, looking for the spending tx in the mempool") - client.getMempool().map { mempoolTxs => - mempoolTxs.filter(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match { - case Nil => + .map(_.flatten) // filter out errors and hint transactions that can't be found + .map(hintTxs => { + hintTxs.find(tx => tx.txIn.exists(i => i.outPoint.txid == w.txId && i.outPoint.index == w.outputIndex)) match { + case Some(spendingTx) => + log.info(s"${w.txId}:${w.outputIndex} has already been spent by a tx provided in hints: txid=${spendingTx.txid}") + context.self ! ProcessNewTransaction(spendingTx) + case None => + // The hints didn't help us, let's search for the spending transaction. + log.info(s"${w.txId}:${w.outputIndex} has already been spent, looking for the spending tx in the mempool") + client.lookForMempoolSpendingTx(w.txId, w.outputIndex).map(Some(_)).recover { case _ => None }.map { + case Some(spendingTx) => + log.info(s"found tx spending ${w.txId}:${w.outputIndex} in the mempool: txid=${spendingTx.txid}") + context.self ! ProcessNewTransaction(spendingTx) + case None => + // no luck, we have to do it the hard way... log.warn(s"${w.txId}:${w.outputIndex} has already been spent, spending tx not in the mempool, looking in the blockchain...") - client.lookForSpendingTx(None, w.txId, w.outputIndex).map { tx => - log.warn(s"found the spending tx of ${w.txId}:${w.outputIndex} in the blockchain: txid=${tx.txid}") - context.self ! ProcessNewTransaction(tx) + client.lookForSpendingTx(None, w.txId, w.outputIndex).map { spendingTx => + log.warn(s"found the spending tx of ${w.txId}:${w.outputIndex} in the blockchain: txid=${spendingTx.txid}") + context.self ! ProcessNewTransaction(spendingTx) } - case txs => - log.info(s"found ${txs.size} txs spending ${w.txId}:${w.outputIndex} in the mempool: txids=${txs.map(_.txid).mkString(",")}") - txs.foreach(tx => context.self ! ProcessNewTransaction(tx)) } - } + } }) } } diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala index 476fd8a77..2df951d9b 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/BitcoinCoreClient.scala @@ -156,6 +156,18 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient) extends OnChainWall } } yield doubleSpent + /** Search for mempool transaction spending a given output. */ + def lookForMempoolSpendingTx(txid: ByteVector32, outputIndex: Int)(implicit ec: ExecutionContext): Future[Transaction] = { + rpcClient.invoke("gettxspendingprevout", Seq(OutpointArg(txid, outputIndex))).collect { + case JArray(results) => results.flatMap(result => (result \ "spendingtxid").extractOpt[String].map(ByteVector32.fromValidHex)) + }.flatMap { spendingTxIds => + spendingTxIds.headOption match { + case Some(spendingTxId) => getTransaction(spendingTxId) + case None => Future.failed(new RuntimeException(s"mempool doesn't contain any transaction spending $txid:$outputIndex")) + } + } + } + /** * Iterate over blocks to find the transaction that has spent a given output. * NB: only call this method when you're sure the output has been spent, otherwise this will iterate over the whole @@ -418,7 +430,7 @@ class BitcoinCoreClient(val rpcClient: BitcoinJsonRPCClient) extends OnChainWall def unlockOutpoints(outPoints: Seq[OutPoint])(implicit ec: ExecutionContext): Future[Boolean] = { // we unlock utxos one by one and not as a list as it would fail at the first utxo that is not actually locked and the rest would not be processed val futures = outPoints - .map(outPoint => UnlockOutpoint(outPoint.txid, outPoint.index)) + .map(outPoint => OutpointArg(outPoint.txid, outPoint.index)) .map(utxo => rpcClient .invoke("lockunspent", true, List(utxo)) .mapTo[JBool] @@ -619,7 +631,8 @@ object BitcoinCoreClient { case class WalletTx(address: String, amount: Satoshi, fees: Satoshi, blockHash: ByteVector32, confirmations: Long, txid: ByteVector32, timestamp: Long) - case class UnlockOutpoint(txid: ByteVector32, vout: Long) + /** Outpoint used as RPC argument. */ + case class OutpointArg(txid: ByteVector32, vout: Long) case class Utxo(txid: ByteVector32, amount: MilliBtc, confirmations: Long, safe: Boolean, label_opt: Option[String]) diff --git a/eclair-core/src/test/resources/integration/bitcoin.conf b/eclair-core/src/test/resources/integration/bitcoin.conf index 20fc3727a..370f1cbfc 100644 --- a/eclair-core/src/test/resources/integration/bitcoin.conf +++ b/eclair-core/src/test/resources/integration/bitcoin.conf @@ -7,7 +7,8 @@ txindex=1 zmqpubhashblock=tcp://127.0.0.1:28334 zmqpubrawtx=tcp://127.0.0.1:28335 rpcworkqueue=64 -fallbackfee=0.0002 +fallbackfee=0.0001 # 10 sat/byte +consolidatefeerate=0 # we don't want bitcoind to consolidate utxos during tests [regtest] bind=127.0.0.1 port=28333 diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala index ed7e8b526..dc1467506 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreClientSpec.scala @@ -1205,6 +1205,9 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A // If we include the mempool, we see that tx1 produces an output that is still unspent. bitcoinClient.isTransactionOutputSpendable(tx1.txid, 0, includeMempool = true).pipeTo(sender.ref) sender.expectMsg(true) + // We're able to find the spending transaction in the mempool. + bitcoinClient.lookForMempoolSpendingTx(tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt).pipeTo(sender.ref) + sender.expectMsg(tx1) // Let's confirm our transaction. generateBlocks(1) @@ -1219,6 +1222,8 @@ class BitcoinCoreClientSpec extends TestKitBaseClass with BitcoindService with A sender.expectMsg(true) generateBlocks(1) + bitcoinClient.lookForMempoolSpendingTx(tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt).pipeTo(sender.ref) + sender.expectMsgType[Failure] bitcoinClient.lookForSpendingTx(None, tx1.txIn.head.outPoint.txid, tx1.txIn.head.outPoint.index.toInt).pipeTo(sender.ref) sender.expectMsg(tx1) } diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala index 14c30c98a..543500a7f 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala @@ -58,7 +58,7 @@ trait BitcoindService extends Logging { val PATH_BITCOIND = sys.env.get("BITCOIND_DIR") match { case Some(customBitcoinDir) => new File(customBitcoinDir, "bitcoind") - case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-23.2/bin/bitcoind") + case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-24.1/bin/bitcoind") } logger.info(s"using bitcoind: $PATH_BITCOIND") val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin") diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/channel/InteractiveTxBuilderSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/channel/InteractiveTxBuilderSpec.scala index 417f9a76b..ed717f70b 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/channel/InteractiveTxBuilderSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/channel/InteractiveTxBuilderSpec.scala @@ -539,9 +539,9 @@ class InteractiveTxBuilderSpec extends TestKitBaseClass with AnyFunSuiteLike wit test("initiator and non-initiator splice-in") { val targetFeerate = FeeratePerKw(1000 sat) val fundingA1 = 100_000 sat - val utxosA = Seq(150_000 sat, 85_000 sat) + val utxosA = Seq(350_000 sat, 150_000 sat) val fundingB1 = 50_000 sat - val utxosB = Seq(90_000 sat, 80_000 sat) + val utxosB = Seq(175_000 sat, 90_000 sat) withFixture(fundingA1, utxosA, fundingB1, utxosB, targetFeerate, 660 sat, 0, RequireConfirmedInputs(forLocal = true, forRemote = true)) { f => import f._ @@ -813,9 +813,9 @@ class InteractiveTxBuilderSpec extends TestKitBaseClass with AnyFunSuiteLike wit test("initiator and non-initiator combine splice-in and splice-out") { val targetFeerate = FeeratePerKw(1000 sat) val fundingA1 = 150_000 sat - val utxosA = Seq(200_000 sat, 100_000 sat) + val utxosA = Seq(480_000 sat, 130_000 sat) val fundingB1 = 100_000 sat - val utxosB = Seq(150_000 sat, 50_000 sat) + val utxosB = Seq(340_000 sat, 70_000 sat) withFixture(fundingA1, utxosA, fundingB1, utxosB, targetFeerate, 660 sat, 0, RequireConfirmedInputs(forLocal = true, forRemote = true)) { f => import f._ @@ -1321,9 +1321,9 @@ class InteractiveTxBuilderSpec extends TestKitBaseClass with AnyFunSuiteLike wit test("fund splice transaction with previous inputs (no new inputs)") { val targetFeerate = FeeratePerKw(2_000 sat) val fundingA1 = 150_000 sat - val utxosA = Seq(200_000 sat, 75_000 sat) + val utxosA = Seq(480_000 sat, 75_000 sat) val fundingB1 = 100_000 sat - val utxosB = Seq(150_000 sat, 50_000 sat) + val utxosB = Seq(325_000 sat, 60_000 sat) withFixture(fundingA1, utxosA, fundingB1, utxosB, targetFeerate, 660 sat, 0, RequireConfirmedInputs(forLocal = false, forRemote = false)) { f => import f._ @@ -1446,9 +1446,9 @@ class InteractiveTxBuilderSpec extends TestKitBaseClass with AnyFunSuiteLike wit test("fund splice transaction with previous inputs (with new inputs)") { val targetFeerate = FeeratePerKw(2_500 sat) val fundingA1 = 100_000 sat - val utxosA = Seq(140_000 sat, 40_000 sat, 35_000 sat) + val utxosA = Seq(340_000 sat, 40_000 sat, 35_000 sat) val fundingB1 = 80_000 sat - val utxosB = Seq(110_000 sat, 20_000 sat, 15_000 sat) + val utxosB = Seq(280_000 sat, 20_000 sat, 15_000 sat) withFixture(fundingA1, utxosA, fundingB1, utxosB, targetFeerate, 660 sat, 0, RequireConfirmedInputs(forLocal = false, forRemote = false)) { f => import f._ @@ -1579,9 +1579,9 @@ class InteractiveTxBuilderSpec extends TestKitBaseClass with AnyFunSuiteLike wit test("funding splice transaction with previous inputs (different balance)") { val targetFeerate = FeeratePerKw(2_500 sat) val fundingA1 = 100_000 sat - val utxosA = Seq(140_000 sat, 40_000 sat, 35_000 sat) + val utxosA = Seq(340_000 sat, 40_000 sat, 35_000 sat) val fundingB1 = 80_000 sat - val utxosB = Seq(110_000 sat, 20_000 sat, 15_000 sat) + val utxosB = Seq(290_000 sat, 20_000 sat, 15_000 sat) withFixture(fundingA1, utxosA, fundingB1, utxosB, targetFeerate, 660 sat, 0, RequireConfirmedInputs(forLocal = false, forRemote = false)) { f => import f._