Upgrade to bitcoin 0.17.1 (#826)

Bitcoin Core 0.18 is about to enter RC cycle and should be release soon (initial target was April). It is not compatible with 0.16 (some of the RPC calls that we use have been removed. They're still available in 0.17 but tagged as deprecated). 

With this PR, eclair will be compatible with 0.17 and the upcoming 0.18, but not with 0.16 any more so it will be a breaking change for some of our users. Supporting the last 2 versions is the right option and we should be ready before 0.18 is actually released (its initial target was April).

eclair-core/pom.xml

@@ -79,10 +79,10 @@
                 <activeByDefault>true</activeByDefault>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-x86_64-linux-gnu.tar.gz
+                <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.17.1/bitcoin-0.17.1-x86_64-linux-gnu.tar.gz
                 </bitcoind.url>
-                <bitcoind.md5>c371e383f024c6c45fb255d528a6beec</bitcoind.md5>
+                <bitcoind.md5>724043999e2b5ed0c088e8db34f15d43</bitcoind.md5>
-                <bitcoind.sha1>e6d8ab1f7661a6654fd81e236b9b5fd35a3d4dcb</bitcoind.sha1>
+                <bitcoind.sha1>546ee35d4089c7ccc040a01cdff3362599b8bc53</bitcoind.sha1>
             </properties>
         </profile>
         <profile>
@@ -93,10 +93,10 @@
                 </os>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-osx64.tar.gz
+                <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.17.1/bitcoin-0.17.1-osx64.tar.gz
                 </bitcoind.url>
-                <bitcoind.md5>bacd87d0c3f65a5acd666e33d094a59e</bitcoind.md5>
+                <bitcoind.md5>b5a792c6142995faa42b768273a493bd</bitcoind.md5>
-                <bitcoind.sha1>62cc5bd9ced610bb9e8d4a854396bfe2139e3d0f</bitcoind.sha1>
+                <bitcoind.sha1>8bd51c7024d71de07df381055993e9f472013db8</bitcoind.sha1>
             </properties>
         </profile>
         <profile>
@@ -107,9 +107,9 @@
                 </os>
             </activation>
             <properties>
-                <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.16.3/bitcoin-0.16.3-win64.zip</bitcoind.url>
+                <bitcoind.url>https://bitcoin.org/bin/bitcoin-core-0.17.1/bitcoin-0.17.1-win64.zip</bitcoind.url>
-                <bitcoind.md5>bbde9b1206956d19298034319e9f405e</bitcoind.md5>
+                <bitcoind.md5>b0e824e9dd02580b5b01f073f3c89858</bitcoind.md5>
-                <bitcoind.sha1>85e3dc8a9c6f93b1b20cb79fa5850b5ce81da221</bitcoind.sha1>
+                <bitcoind.sha1>4e17bad7d08c465b444143a93cd6eb1c95076e3f</bitcoind.sha1>
             </properties>
         </profile>
     </profiles>

eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala

@@ -126,15 +126,13 @@ class Setup(datadir: File,
       } yield (progress, chainHash, bitcoinVersion, unspentAddresses, blocks, headers)
       // blocking sanity checks
       val (progress, chainHash, bitcoinVersion, unspentAddresses, blocks, headers) = await(future, 30 seconds, "bicoind did not respond after 30 seconds")
-      assert(bitcoinVersion >= 160300, "Eclair requires Bitcoin Core 0.16.3 or higher")
+      assert(bitcoinVersion >= 170000, "Eclair requires Bitcoin Core 0.17.0 or higher")
       assert(chainHash == nodeParams.chainHash, s"chainHash mismatch (conf=${nodeParams.chainHash} != bitcoind=$chainHash)")
       if (chainHash != Block.RegtestGenesisBlock.hash) {
         assert(unspentAddresses.forall(address => !isPay2PubkeyHash(address)), "Make sure that all your UTXOS are segwit UTXOS and not p2pkh (check out our README for more details)")
       }
       assert(progress > 0.999, s"bitcoind should be synchronized (progress=$progress")
       assert(headers - blocks <= 1, s"bitcoind should be synchronized (headers=$headers blocks=$blocks")
-      // TODO: add a check on bitcoin version?
-
       Bitcoind(bitcoinClient)
     case ELECTRUM =>
       val addresses = config.hasPath("electrum") match {

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreWallet.scala

@@ -19,10 +19,12 @@ package fr.acinq.eclair.blockchain.bitcoind
 import fr.acinq.bitcoin._
 import fr.acinq.eclair._
 import fr.acinq.eclair.blockchain._
-import fr.acinq.eclair.blockchain.bitcoind.rpc.{BitcoinJsonRPCClient, JsonRPCError}
+import fr.acinq.eclair.blockchain.bitcoind.rpc.{BitcoinJsonRPCClient, Error, JsonRPCError}
 import fr.acinq.eclair.transactions.Transactions
 import grizzled.slf4j.Logging
+import org.json4s.DefaultFormats
 import org.json4s.JsonAST._
+import org.json4s.jackson.Serialization
 import scodec.bits.ByteVector
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -47,9 +49,13 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient)(implicit ec: ExecutionC
   def fundTransaction(tx: Transaction, lockUnspents: Boolean, feeRatePerKw: Long): Future[FundTransactionResponse] = fundTransaction(Transaction.write(tx).toHex, lockUnspents, feeRatePerKw)
 
   def signTransaction(hex: String): Future[SignTransactionResponse] =
-    rpcClient.invoke("signrawtransaction", hex).map(json => {
+    rpcClient.invoke("signrawtransactionwithwallet", hex).map(json => {
       val JString(hex) = json \ "hex"
       val JBool(complete) = json \ "complete"
+      if (!complete) {
+        val message = (json \ "errors" \\ classOf[JString]).mkString(",")
+        throw new JsonRPCError(Error(-1, message))
+      }
       SignTransactionResponse(Transaction.read(hex), complete)
     })
 
@@ -74,7 +80,7 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient)(implicit ec: ExecutionC
 
   private def signTransactionOrUnlock(tx: Transaction): Future[SignTransactionResponse] = {
     val f = signTransaction(tx)
-    // if signature fails (e.g. because wallet is uncrypted) we need to unlock the utxos
+    // if signature fails (e.g. because wallet is encrypted) we need to unlock the utxos
     f.recoverWith { case _ =>
       unlockOutpoints(tx.txIn.map(_.outPoint))
         .recover { case t: Throwable => logger.warn(s"Cannot unlock failed transaction's UTXOs txid=${tx.txid}", t); t } // no-op, just add a log in case of failure
@@ -94,7 +100,7 @@ class BitcoinCoreWallet(rpcClient: BitcoinJsonRPCClient)(implicit ec: ExecutionC
       // we ask bitcoin core to add inputs to the funding tx, and use the specified change address
       FundTransactionResponse(unsignedFundingTx, _, fee) <- fundTransaction(partialFundingTx, lockUnspents = true, feeRatePerKw)
       // now let's sign the funding tx
-      SignTransactionResponse(fundingTx, _) <- signTransactionOrUnlock(unsignedFundingTx)
+      SignTransactionResponse(fundingTx, true) <- signTransactionOrUnlock(unsignedFundingTx)
       // there will probably be a change output, so we need to find which output is ours
       outputIndex = Transactions.findPubKeyScriptIndex(fundingTx, pubkeyScript, outputsAlreadyUsed = Set.empty, amount_opt = None)
       _ = logger.debug(s"created funding txid=${fundingTx.txid} outputIndex=$outputIndex fee=$fee")

eclair-core/src/test/resources/integration/bitcoin.conf

@@ -1,7 +1,7 @@
 regtest=1
+noprinttoconsole=1 
 server=1
 port=28333
-rpcport=28332
 rpcuser=foo
 rpcpassword=bar
 txindex=1
@@ -9,3 +9,5 @@ zmqpubrawblock=tcp://127.0.0.1:28334
 zmqpubrawtx=tcp://127.0.0.1:28335
 rpcworkqueue=64
 addresstype=bech32
+[regtest]
+rpcport=28332

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoinCoreWalletSpec.scala

@@ -21,7 +21,7 @@ import akka.actor.Status.Failure
 import akka.pattern.pipe
 import akka.testkit.{TestKit, TestProbe}
 import com.typesafe.config.ConfigFactory
-import fr.acinq.bitcoin.{Block, MilliBtc, Satoshi, Script, Transaction}
+import fr.acinq.bitcoin.{ByteVector32, Block, MilliBtc, OutPoint, Satoshi, Script, Transaction, TxIn, TxOut}
 import fr.acinq.eclair.blockchain._
 import fr.acinq.eclair.blockchain.bitcoind.BitcoinCoreWallet.FundTransactionResponse
 import fr.acinq.eclair.blockchain.bitcoind.rpc.{BasicBitcoinJsonRPCClient, JsonRPCError}
@@ -41,7 +41,14 @@ import scala.util.{Random, Try}
 
 class BitcoinCoreWalletSpec extends TestKit(ActorSystem("test")) with BitcoindService with FunSuiteLike with BeforeAndAfterAll with Logging {
 
-  val commonConfig = ConfigFactory.parseMap(Map("eclair.chain" -> "regtest", "eclair.spv" -> false, "eclair.server.public-ips.1" -> "localhost", "eclair.bitcoind.port" -> 28333, "eclair.bitcoind.rpcport" -> 28332, "eclair.bitcoind.zmq" -> "tcp://127.0.0.1:28334", "eclair.router-broadcast-interval" -> "2 second", "eclair.auto-reconnect" -> false))
+  val commonConfig = ConfigFactory.parseMap(Map(
+    "eclair.chain" -> "regtest",
+    "eclair.spv" -> false,
+    "eclair.server.public-ips.1" -> "localhost",
+    "eclair.bitcoind.port" -> 28333,
+    "eclair.bitcoind.rpcport" -> 28332,
+    "eclair.router-broadcast-interval" -> "2 second",
+    "eclair.auto-reconnect" -> false))
   val config = ConfigFactory.load(commonConfig).getConfig("eclair")
 
   val walletPassword = Random.alphanumeric.take(8).mkString
@@ -94,6 +101,39 @@ class BitcoinCoreWalletSpec extends TestKit(ActorSystem("test")) with BitcoindSe
     }
   }
 
+  test("handle errors when signing transactions") {
+    val bitcoinClient = new BasicBitcoinJsonRPCClient(
+      user = config.getString("bitcoind.rpcuser"),
+      password = config.getString("bitcoind.rpcpassword"),
+      host = config.getString("bitcoind.host"),
+      port = config.getInt("bitcoind.rpcport"))
+    val wallet = new BitcoinCoreWallet(bitcoinClient)
+
+    val sender = TestProbe()
+
+    // create a transaction that spends UTXOs that don't exist
+    wallet.getFinalAddress.pipeTo(sender.ref)
+    val address = sender.expectMsgType[String]
+    val unknownTxids = Seq(
+      ByteVector32.fromValidHex("01" * 32),
+      ByteVector32.fromValidHex("02" * 32),
+      ByteVector32.fromValidHex("03" * 32)
+    )
+    val unsignedTx = Transaction(version = 2,
+      txIn = Seq(
+        TxIn(OutPoint(unknownTxids(0), 0), signatureScript = Nil, sequence = TxIn.SEQUENCE_FINAL),
+        TxIn(OutPoint(unknownTxids(1), 0), signatureScript = Nil, sequence = TxIn.SEQUENCE_FINAL),
+        TxIn(OutPoint(unknownTxids(2), 0), signatureScript = Nil, sequence = TxIn.SEQUENCE_FINAL)
+      ),
+      txOut = TxOut(Satoshi(1000000), addressToPublicKeyScript(address, Block.RegtestGenesisBlock.hash)) :: Nil,
+      lockTime = 0)
+
+    // signing it should fail, and the error message should contain the txids of the UTXOs that could not be used
+    wallet.signTransaction(unsignedTx).pipeTo(sender.ref)
+    val Failure(JsonRPCError(error)) = sender.expectMsgType[Failure]
+    unknownTxids.foreach(id => assert(error.message.contains(id.toString())))
+  }
+
   test("create/commit/rollback funding txes") {
     val bitcoinClient = new BasicBitcoinJsonRPCClient(
       user = config.getString("bitcoind.rpcuser"),
@@ -141,10 +181,9 @@ class BitcoinCoreWalletSpec extends TestKit(ActorSystem("test")) with BitcoindSe
     sender.send(bitcoincli, BitcoinReq("getrawtransaction", fundingTxes(2).txid.toString()))
     assert(sender.expectMsgType[JString](10 seconds).s === fundingTxes(2).toString())
 
-    // NB: bitcoin core doesn't clear the locks when a tx is published
+    // NB: from 0.17.0 on bitcoin core will clear locks when a tx is published
     sender.send(bitcoincli, BitcoinReq("listlockunspent"))
-    assert(sender.expectMsgType[JValue](10 seconds).children.size === 2)
+    assert(sender.expectMsgType[JValue](10 seconds).children.size === 0)
-
   }
 
   test("encrypt wallet") {
@@ -177,7 +216,8 @@ class BitcoinCoreWalletSpec extends TestKit(ActorSystem("test")) with BitcoindSe
 
     val pubkeyScript = Script.write(Script.pay2wsh(Scripts.multiSig2of2(randomKey.publicKey, randomKey.publicKey)))
     wallet.makeFundingTx(pubkeyScript, MilliBtc(50), 10000).pipeTo(sender.ref)
-    assert(sender.expectMsgType[Failure].cause.asInstanceOf[JsonRPCError].error.message.contains("Please enter the wallet passphrase with walletpassphrase first."))
+    val error = sender.expectMsgType[Failure].cause.asInstanceOf[JsonRPCError].error
+    assert(error.message.contains("Please enter the wallet passphrase with walletpassphrase first"))
 
     sender.send(bitcoincli, BitcoinReq("listlockunspent"))
     assert(sender.expectMsgType[JValue](10 seconds).children.size === 0)
@@ -212,14 +252,14 @@ class BitcoinCoreWalletSpec extends TestKit(ActorSystem("test")) with BitcoindSe
     bitcoinClient.invoke("fundrawtransaction", noinputTx1).pipeTo(sender.ref)
     val json = sender.expectMsgType[JValue]
     val JString(unsignedtx1) = json \ "hex"
-    bitcoinClient.invoke("signrawtransaction", unsignedtx1).pipeTo(sender.ref)
+    bitcoinClient.invoke("signrawtransactionwithwallet", unsignedtx1).pipeTo(sender.ref)
     val JString(signedTx1) = sender.expectMsgType[JValue] \ "hex"
     val tx1 = Transaction.read(signedTx1)
     // let's then generate another tx that double spends the first one
     val inputs = tx1.txIn.map(txIn => Map("txid" -> txIn.outPoint.txid.toString, "vout" -> txIn.outPoint.index)).toArray
     bitcoinClient.invoke("createrawtransaction", inputs, Map(address -> tx1.txOut.map(_.amount.toLong).sum * 1.0 / 1e8)).pipeTo(sender.ref)
     val JString(unsignedtx2) = sender.expectMsgType[JValue]
-    bitcoinClient.invoke("signrawtransaction", unsignedtx2).pipeTo(sender.ref)
+    bitcoinClient.invoke("signrawtransactionwithwallet", unsignedtx2).pipeTo(sender.ref)
     val JString(signedTx2) = sender.expectMsgType[JValue] \ "hex"
     val tx2 = Transaction.read(signedTx2)
 
@@ -236,4 +276,4 @@ class BitcoinCoreWalletSpec extends TestKit(ActorSystem("test")) with BitcoindSe
     sender.expectMsg(true)
   }
 
-}
+}

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala

@@ -44,7 +44,7 @@ trait BitcoindService extends Logging {
   val INTEGRATION_TMP_DIR = new File(TestUtils.BUILD_DIRECTORY, s"integration-${UUID.randomUUID()}")
   logger.info(s"using tmp dir: $INTEGRATION_TMP_DIR")
 
-  val PATH_BITCOIND = new File(TestUtils.BUILD_DIRECTORY, "bitcoin-0.16.3/bin/bitcoind")
+  val PATH_BITCOIND = new File(TestUtils.BUILD_DIRECTORY, "bitcoin-0.17.1/bin/bitcoind")
   val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin")
 
   var bitcoind: Process = null

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/ExtendedBitcoinClientSpec.scala

@@ -34,7 +34,14 @@ import scala.concurrent.ExecutionContext.Implicits.global
 
 class ExtendedBitcoinClientSpec extends TestKit(ActorSystem("test")) with BitcoindService with FunSuiteLike with BeforeAndAfterAll with Logging {
 
-  val commonConfig = ConfigFactory.parseMap(Map("eclair.chain" -> "regtest", "eclair.spv" -> false, "eclair.server.public-ips.1" -> "localhost", "eclair.bitcoind.port" -> 28333, "eclair.bitcoind.rpcport" -> 28332, "eclair.bitcoind.zmq" -> "tcp://127.0.0.1:28334", "eclair.router-broadcast-interval" -> "2 second", "eclair.auto-reconnect" -> false))
+  val commonConfig = ConfigFactory.parseMap(Map(
+    "eclair.chain" -> "regtest",
+    "eclair.spv" -> false,
+    "eclair.server.public-ips.1" -> "localhost",
+    "eclair.bitcoind.port" -> 28333,
+    "eclair.bitcoind.rpcport" -> 28332,
+    "eclair.router-broadcast-interval" -> "2 second",
+    "eclair.auto-reconnect" -> false))
   val config = ConfigFactory.load(commonConfig).getConfig("eclair")
 
   implicit val formats = DefaultFormats
@@ -67,7 +74,7 @@ class ExtendedBitcoinClientSpec extends TestKit(ActorSystem("test")) with Bitcoi
     val json = sender.expectMsgType[JValue]
     val JString(unsignedtx) = json \ "hex"
     val JInt(changePos) = json \ "changepos"
-    bitcoinClient.invoke("signrawtransaction", unsignedtx).pipeTo(sender.ref)
+    bitcoinClient.invoke("signrawtransactionwithwallet", unsignedtx).pipeTo(sender.ref)
     val JString(signedTx) = sender.expectMsgType[JValue] \ "hex"
     val tx = Transaction.read(signedTx)
     val txid = tx.txid.toString()
@@ -92,7 +99,7 @@ class ExtendedBitcoinClientSpec extends TestKit(ActorSystem("test")) with Bitcoi
       val pos = if (changePos == 0) 1 else 0
       bitcoinClient.invoke("createrawtransaction", Array(Map("txid" -> txid, "vout" -> pos)), Map(address -> 5.99999)).pipeTo(sender.ref)
       val JString(unsignedtx) = sender.expectMsgType[JValue]
-      bitcoinClient.invoke("signrawtransaction", unsignedtx).pipeTo(sender.ref)
+      bitcoinClient.invoke("signrawtransactionwithwallet", unsignedtx).pipeTo(sender.ref)
       val JString(signedTx) = sender.expectMsgType[JValue] \ "hex"
       signedTx
     }

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/fee/BitcoinCoreFeeProviderSpec.scala

@@ -38,7 +38,14 @@ import scala.util.Random
 
 class BitcoinCoreFeeProviderSpec extends TestKit(ActorSystem("test")) with BitcoindService with FunSuiteLike with BeforeAndAfterAll with Logging {
 
-  val commonConfig = ConfigFactory.parseMap(Map("eclair.chain" -> "regtest", "eclair.spv" -> false, "eclair.server.public-ips.1" -> "localhost", "eclair.bitcoind.port" -> 28333, "eclair.bitcoind.rpcport" -> 28332, "eclair.bitcoind.zmq" -> "tcp://127.0.0.1:28334", "eclair.router-broadcast-interval" -> "2 second", "eclair.auto-reconnect" -> false))
+  val commonConfig = ConfigFactory.parseMap(Map(
+    "eclair.chain" -> "regtest",
+    "eclair.spv" -> false,
+    "eclair.server.public-ips.1" -> "localhost",
+    "eclair.bitcoind.port" -> 28333,
+    "eclair.bitcoind.rpcport" -> 28332,
+    "eclair.router-broadcast-interval" -> "2 second",
+    "eclair.auto-reconnect" -> false))
   val config = ConfigFactory.load(commonConfig).getConfig("eclair")
 
   val walletPassword = Random.alphanumeric.take(8).mkString

travis/builddeps.sh

@@ -1,25 +0,0 @@
-pushd .
-# lightning deps
-sudo add-apt-repository -y ppa:chris-lea/libsodium
-sudo apt-get update
-sudo apt-get install -y libsodium-dev libgmp-dev libsqlite3-dev
-cd
-git clone https://github.com/luke-jr/libbase58.git
-cd libbase58
-./autogen.sh && ./configure && make && sudo make install
-# lightning
-cd
-git clone https://github.com/ElementsProject/lightning.git
-cd lightning
-git checkout fce9ee29e3c37b4291ebb050e6a687cfaa7df95a
-git submodule init
-git submodule update
-make
-# bitcoind
-cd
-wget https://bitcoin.org/bin/bitcoin-core-0.13.0/bitcoin-0.13.0-x86_64-linux-gnu.tar.gz
-echo "bcc1e42d61f88621301bbb00512376287f9df4568255f8b98bc10547dced96c8  bitcoin-0.13.0-x86_64-linux-gnu.tar.gz" > sha256sum.asc
-sha256sum -c sha256sum.asc
-tar xzvf bitcoin-0.13.0-x86_64-linux-gnu.tar.gz
-popd
-