Add nightly build with bitcoind master (#2027)

We should regularly run against bitcoind master, to detect if a change to
bitcoind is affecting us.

This will let us detect changes that may break lightning before bitcoind
releases them, giving us a chance to rectify it.

We really don't want to run this workflow for every pull request or every
merge to master, we instead run it twice per week, which should give
us enough time to detect dangerous changes.

.github/workflows/latest-bitcoind.yml

@@ -0,0 +1,51 @@
+name: Latest Bitcoin Core
+
+on:
+  schedule:
+    # Run at midnight on Sunday and Wednesday.
+    - cron: '0 0 * * 0,3'
+
+jobs:
+
+  regression-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout bitcoind master
+        uses: actions/checkout@v2
+        with:
+          repository: bitcoin/bitcoin
+          path: bitcoin
+
+      - name: Install bitcoind dependencies
+        run: sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils python3 libssl-dev libevent-dev libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libminiupnpc-dev libzmq3-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libprotobuf-dev protobuf-compiler git libsqlite3-dev ccache
+        working-directory: ./bitcoin
+
+      - name: Autogen bitcoind
+        run: ./autogen.sh
+        working-directory: ./bitcoin
+
+      - name: Configure bitcoind
+        run: ./configure --with-zmq --without-gui --disable-shared --with-pic --disable-tests --disable-bench
+        working-directory: ./bitcoin
+
+      - name: Build bitcoind
+        run: make -j "$(($(nproc)))"
+        working-directory: ./bitcoin
+
+      - name: Checkout eclair master
+        uses: actions/checkout@v2
+        with:
+          path: eclair
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+
+      - name: Configure OS settings
+        run: echo "fs.file-max = 1024000" | sudo tee -a /etc/sysctl.conf
+
+      - name: Run eclair tests
+        run: BITCOIND_DIR=$GITHUB_WORKSPACE/bitcoin/src mvn test
+        working-directory: ./eclair

BUILD.md

@@ -45,6 +45,12 @@ To run tests for a specific class, run:
 mvn test -Dsuites=*<TestClassName>
 ```
 
+To run tests with a specific version of `bitcoind`, run:
+
+```shell
+BITCOIND_DIR=<absolute/path/to/directory> mvn test
+```
+
 ### Build specific module
 
 To only build the `eclair-node` module, run:

eclair-core/src/test/scala/fr/acinq/eclair/TestUtils.scala

@@ -18,8 +18,8 @@ package fr.acinq.eclair
 
 import akka.actor.ActorRef
 import akka.event.DiagnosticLoggingAdapter
-import akka.testkit
 import akka.testkit.{TestActor, TestProbe}
+import fr.acinq.eclair.channel.Channel
 import fr.acinq.eclair.io.Peer
 import fr.acinq.eclair.wire.protocol.LightningMessage
 
@@ -31,8 +31,8 @@ import java.util.UUID
 object TestUtils {
 
   /**
-    * Get the module's target directory (works from command line and within intellij)
-    */
+   * Get the module's target directory (works from command line and within intellij)
+   */
   val BUILD_DIRECTORY = sys
     .props
     .get("buildDirectory") // this is defined if we run from maven
@@ -53,19 +53,19 @@ object TestUtils {
   def newIntegrationTmpDir(baseDir: String = TestUtils.BUILD_DIRECTORY) = new File(baseDir, s"integration-${UUID.randomUUID()}")
 
   object NoLoggingDiagnostics extends DiagnosticLoggingAdapter {
+    // @formatter:off
     override def isErrorEnabled: Boolean = false
     override def isWarningEnabled: Boolean = false
     override def isInfoEnabled: Boolean = false
     override def isDebugEnabled: Boolean = false
-
     override protected def notifyError(message: String): Unit = ()
     override protected def notifyError(cause: Throwable, message: String): Unit = ()
     override protected def notifyWarning(message: String): Unit = ()
     override protected def notifyInfo(message: String): Unit = ()
     override protected def notifyDebug(message: String): Unit = ()
+    // @formatter:on
   }
 
-
   /**
    * [[Channel]] encapsulates outgoing messages in [[Peer.OutgoingMessage]] due to how connection management works.
    *
@@ -73,13 +73,11 @@ object TestUtils {
    * easier. You can now pass a [[TestProbe]] as a connection and only deal with incoming/outgoing [[LightningMessage]].
    */
   def forwardOutgoingToPipe(peer: TestProbe, pipe: ActorRef): Unit = {
-    peer.setAutoPilot(new testkit.TestActor.AutoPilot {
-      override def run(sender: ActorRef, msg: Any): TestActor.AutoPilot = msg match {
-        case Peer.OutgoingMessage(msg: LightningMessage, _: ActorRef) =>
-          pipe.tell(msg, sender)
-          TestActor.KeepRunning
-        case _ => TestActor.KeepRunning
-      }
+    peer.setAutoPilot((sender: ActorRef, msg: Any) => msg match {
+      case Peer.OutgoingMessage(msg: LightningMessage, _: ActorRef) =>
+        pipe.tell(msg, sender)
+        TestActor.KeepRunning
+      case _ => TestActor.KeepRunning
     })
   }
 

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala

@@ -54,7 +54,11 @@ trait BitcoindService extends Logging {
   val INTEGRATION_TMP_DIR: File = TestUtils.newIntegrationTmpDir()
   logger.info(s"using tmp dir: $INTEGRATION_TMP_DIR")
 
-  val PATH_BITCOIND = new File(TestUtils.BUILD_DIRECTORY, "bitcoin-0.21.1/bin/bitcoind")
+  val PATH_BITCOIND = sys.env.get("BITCOIND_DIR") match {
+    case Some(customBitcoinDir) => new File(customBitcoinDir, "bitcoind")
+    case None => new File(TestUtils.BUILD_DIRECTORY, "bitcoin-0.21.1/bin/bitcoind")
+  }
+  logger.info(s"using bitcoind: $PATH_BITCOIND")
   val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin")
 
   var bitcoind: Process = _

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/fee/BitcoinCoreFeeProviderSpec.scala

@@ -16,7 +16,6 @@
 
 package fr.acinq.eclair.blockchain.fee
 
-import akka.actor.Status.Failure
 import akka.pattern.pipe
 import akka.testkit.TestProbe
 import fr.acinq.bitcoin._
@@ -67,12 +66,7 @@ class BitcoinCoreFeeProviderSpec extends TestKitBaseClass with BitcoindService w
   }
 
   test("get fee rates") {
-    // the regtest client doesn't have enough data to estimate fees yet, so it's supposed to fail
-    val regtestProvider = new BitcoinCoreFeeProvider(bitcoinrpcclient, FeeratesPerKB(FeeratePerKB(1 sat), FeeratePerKB(1 sat), FeeratePerKB(2 sat), FeeratePerKB(3 sat), FeeratePerKB(4 sat), FeeratePerKB(5 sat), FeeratePerKB(6 sat), FeeratePerKB(7 sat), FeeratePerKB(8 sat)))
     val sender = TestProbe()
-    regtestProvider.getFeerates.pipeTo(sender.ref)
-    assert(sender.expectMsgType[Failure].cause.asInstanceOf[RuntimeException].getMessage.contains("Insufficient data or no feerate found"))
-
     val fees = Map(
       1 -> FeeratePerKB(1500 sat),
       2 -> FeeratePerKB(1400 sat),
@@ -93,7 +87,8 @@ class BitcoinCoreFeeProviderSpec extends TestKitBaseClass with BitcoindService w
       blocks_36 = fees(36),
       blocks_72 = fees(72),
       blocks_144 = fees(144),
-      blocks_1008 = fees(1008))
+      blocks_1008 = fees(1008)
+    )
 
     val mockBitcoinClient = new BasicBitcoinJsonRPCClient(rpcAuthMethod = UserPassword("", ""), host = "localhost", port = 0) {
       override def invoke(method: String, params: Any*)(implicit ec: ExecutionContext): Future[JValue] = method match {