Build and CI improvements (#710)

* Build and CI improvements

In this commit we:

1) Parallelize the Travis CI config, by splitting
    each project into its own Travis task
2) Download bitcoind binaries through sbt

* Use binaries downloaded by sbt task

* Make BitcoindRpcTestUtil work on Travis without bitcoind on PATH

* Add new downloadEclair task to sbt

* use sbt downloaded binaries in tests

* Fix Eclair and Bitcoind tests

.gitignore

@@ -1,6 +1,10 @@
 *.class
 *.log
 
+# binaries downloaded by sbt for tests
+binaries/bitcoind
+binaries/eclair
+
 # sbt specific
 .cache
 .history
@@ -72,4 +76,4 @@ libsecp256k1.pc
 # Docusaurs
 node_modules
 website/build
-website/static/api
+website/static/api

bitcoind-rpc-test/bitcoind-rpc-test.sbt

@@ -0,0 +1,12 @@
+name := "bitcoin-s-bitcoind-rpc-test"
+
+libraryDependencies ++= Deps.bitcoindRpcTest(scalaVersion.value)
+
+lazy val downloadBitcoind = taskKey[Unit] {
+  "Download bitcoind binaries, extract to ./bitcoind-binaries"
+}
+
+import java.nio.file.Paths
+lazy val bitcoindRpc = project in Paths.get("..", "bitcoind-rpc").toFile
+
+Test / test := (Test / test dependsOn bitcoindRpc / downloadBitcoind).value

bitcoind-rpc/README.md

@@ -1,21 +1,2 @@
-See the `bitcoind`/Bitcoin Core section on the 
+See the `bitcoind`/Bitcoin Core section on the
 Bitcoin-S [website](https://bitcoin-s.org/docs/rpc/rpc-bitcoind).
-
-## Testing
-
-To test the Bitcoin-S RPC project you need both version 0.16 and 0.17 of Bitcoin Core. A list of current and previous releases can be found [here](https://bitcoincore.org/en/releases/).
-
-You then need to set environment variables to indicate where Bitcoin-S can find the different versions: 
-
-```bash
-$ export BITCOIND_V16_PATH=/path/to/v16/bitcoind
-$ export BITCOIND_V17_PATH=/path/to/v17/bitcoind
-```
-
-If you just run tests testing common functionality it's enough to have either version 0.16 or 0.17 on your `PATH`.
-
-To run all RPC related tests:
-
-```bash
-$ bloop test bitcoindRpcTest
-```

bitcoind-rpc/bitcoind-rpc.sbt

@@ -0,0 +1,72 @@
+import scala.util.Properties
+import scala.collection.JavaConverters._
+import java.nio.file.Files
+import java.nio.file.Paths
+
+name := "bitcoin-s-bitcoind-rpc"
+
+libraryDependencies ++= Deps.bitcoindRpc
+
+dependsOn {
+  lazy val core = project in Paths.get("..", "core").toFile
+  core
+}
+
+lazy val downloadBitcoind = taskKey[Unit] {
+  "Download bitcoind binaries, extract to ./binaries/bitcoind"
+}
+
+downloadBitcoind := {
+  val logger = streams.value.log
+  import scala.sys.process._
+
+  val binaryDir = Paths.get("binaries", "bitcoind")
+
+  if (Files.notExists(binaryDir)) {
+    logger.info(s"Creating directory for bitcoind binaries: $binaryDir")
+    Files.createDirectories(binaryDir)
+  }
+
+  val versions = List("0.17.0.1", "0.16.3")
+
+  logger.debug(
+    s"(Maybe) downloading Bitcoin Core binaries for versions: ${versions.mkString(",")}")
+
+  val platform =
+    if (Properties.isLinux) "x86_64-linux-gnu"
+    else if (Properties.isMac) "osx64"
+    else sys.error(s"Unsupported OS: ${Properties.osName}")
+
+  versions.foreach { version =>
+    val versionDir = binaryDir resolve version
+    val archiveLocation = binaryDir resolve s"$version.tar.gz"
+    val location =
+      s"https://bitcoincore.org/bin/bitcoin-core-$version/bitcoin-$version-$platform.tar.gz"
+
+    val expectedEndLocation = binaryDir resolve s"bitcoin-$version"
+    if (Files
+          .list(binaryDir)
+          .iterator
+          .asScala
+          .map(_.toString)
+          .exists(expectedEndLocation.toString.startsWith(_))) {
+      logger.debug(
+        s"Directory $expectedEndLocation already exists, skipping download of version $version")
+    } else {
+      logger.info(
+        s"Downloading bitcoind version $version from location: $location")
+      logger.info(s"Placing the file in $archiveLocation")
+      val downloadCommand = url(location) #> archiveLocation.toFile
+      downloadCommand.!!
+
+      logger.info(s"Download complete, unzipping result")
+
+      val extractCommand = s"tar -xzf $archiveLocation --directory $binaryDir"
+      logger.info(s"Extracting archive with command: $extractCommand")
+      extractCommand.!!
+
+      logger.info(s"Deleting archive")
+      Files.delete(archiveLocation)
+    }
+  }
+}

bitcoind-rpc/src/main/scala/org/bitcoins/rpc/client/common/BitcoindRpcClient.scala

@@ -60,6 +60,9 @@ sealed trait BitcoindVersion
 
 object BitcoindVersion {
 
+  /** The newest `bitcoind` version supported by Bitcoin-S */
+  val newest = V17
+
   case object V16 extends BitcoindVersion {
     override def toString: String = "v0.16"
   }

build.sbt

@@ -382,15 +382,10 @@ lazy val zmq = project
 lazy val bitcoindRpc = project
   .in(file("bitcoind-rpc"))
   .settings(commonProdSettings: _*)
-  .settings(name := "bitcoin-s-bitcoind-rpc",
-            libraryDependencies ++= Deps.bitcoindRpc)
-  .dependsOn(core)
 
 lazy val bitcoindRpcTest = project
   .in(file("bitcoind-rpc-test"))
   .settings(commonTestSettings: _*)
-  .settings(libraryDependencies ++= Deps.bitcoindRpcTest(scalaVersion.value),
-            name := "bitcoin-s-bitcoind-rpc-test")
   .dependsOn(core % testAndCompile, testkit)
 
 lazy val bench = project
@@ -406,12 +401,6 @@ lazy val bench = project
 lazy val eclairRpc = project
   .in(file("eclair-rpc"))
   .settings(commonProdSettings: _*)
-  .settings(name := "bitcoin-s-eclair-rpc",
-            libraryDependencies ++= Deps.eclairRpc)
-  .dependsOn(
-    core,
-    bitcoindRpc
-  )
 
 lazy val eclairRpcTest = project
   .in(file("eclair-rpc-test"))

docs/rpc/bitcoind.md

@@ -124,22 +124,3 @@ val txid: Future[DoubleSha256DigestBE] =
       }
   }
 ```
-
-## Testing
-
-To test the Bitcoin-S RPC project you need both version 0.16 and 0.17 of Bitcoin Core. A list of current and previous releases can be found [here](https://bitcoincore.org/en/releases/).
-
-You then need to set environment variables to indicate where Bitcoin-S can find the different versions:
-
-```bash
-$ export BITCOIND_V16_PATH=/path/to/v16/bitcoind
-$ export BITCOIND_V17_PATH=/path/to/v17/bitcoind
-```
-
-If you just run tests testing common functionality it's enough to have either version 0.16 or 0.17 on your `PATH`.
-
-To run all RPC related tests:
-
-```bash
-$ bash sbt bitcoindRpcTest/test
-```

docs/rpc/eclair.md

@@ -22,6 +22,7 @@ To run Eclair you can use this command:
 $ java -jar eclair-node-0.2-beta8-52821b8.jar &
 ```
 
-Alternatively you can set the `ECLAIR_PATH` env variable and then you can start Eclair with the `start` method on `EclairRpcClient`.
+If you wish to start Eclair from the RPC client, you can do one of the following:
 
-**YOU NEED TO SET `ECLAIR_PATH` CORRECTLY TO BE ABLE TO RUN THE UNIT TESTS**
+1. Construct a `EclairRpcClient` with the `binary` field set
+2. Set the `ECLAIR_PATH` environment variable to the directory where the Eclair Jar is located.

eclair-rpc-test/eclair-rpc-test.sbt

@@ -0,0 +1,8 @@
+lazy val downloadEclair = taskKey[Unit] {
+  "Download Eclair binaries, extract ./binaries/eclair"
+}
+
+import java.nio.file.Paths
+lazy val eclairRpc = project in Paths.get("..", "eclair-rpc").toFile
+
+Test / test := (Test / test dependsOn eclairRpc / downloadEclair).value

eclair-rpc-test/src/test/scala/org/bitcoins/eclair/rpc/EclairRpcClientTest.scala

@@ -37,9 +37,28 @@ import org.bitcoins.core.protocol.ln.{
 import org.bitcoins.testkit.async.TestAsyncUtil
 
 import scala.concurrent.duration._
+import java.nio.file.Files
 
 class EclairRpcClientTest extends AsyncFlatSpec with BeforeAndAfterAll {
 
+  private val dirExists = Files.exists(EclairRpcTestUtil.binaryDirectory)
+  private val hasContents = dirExists && Files
+    .list(EclairRpcTestUtil.binaryDirectory)
+    .toArray()
+    .nonEmpty
+
+  if (!hasContents) {
+    import System.err.{println => printerr}
+    printerr()
+    printerr(s"Run 'sbt downloadEclair' to fetch needed binaries")
+    sys.error {
+      val msg =
+        s""""Eclair binary directory (${BitcoindRpcTestUtil.binaryDirectory}) is empty. 
+        |Run 'sbt downloadEclair' to fetch needed binaries""".stripMargin
+      msg
+    }
+  }
+
   implicit val system: ActorSystem =
     ActorSystem("EclairRpcClient", BitcoindRpcTestUtil.AKKA_CONFIG)
   implicit val m: ActorMaterializer = ActorMaterializer.create(system)
@@ -326,7 +345,7 @@ class EclairRpcClientTest extends AsyncFlatSpec with BeforeAndAfterAll {
       bitcoind <- EclairRpcTestUtil.startedBitcoindRpcClient()
       eclair <- {
         val server = EclairRpcTestUtil.eclairInstance(bitcoind)
-        val eclair = new EclairRpcClient(server)
+        val eclair = new EclairRpcClient(server, EclairRpcTestUtil.binary)
         eclair.start().map(_ => eclair)
       }
       _ <- TestAsyncUtil.retryUntilSatisfiedF(conditionF =
@@ -451,7 +470,8 @@ class EclairRpcClientTest extends AsyncFlatSpec with BeforeAndAfterAll {
       executeWithClientOtherClient(getBadInstance)
     }
 
-    val badClientF = badInstanceF.map(new EclairRpcClient(_))
+    val badClientF =
+      badInstanceF.map(new EclairRpcClient(_, EclairRpcTestUtil.binary))
 
     badClientF.flatMap { badClient =>
       recoverToSucceededIf[RuntimeException](badClient.getInfo)

eclair-rpc/eclair-rpc.sbt

@@ -0,0 +1,50 @@
+import java.nio.file._
+
+name := "bitcoin-s-eclair-rpc"
+
+libraryDependencies ++= Deps.eclairRpc
+
+dependsOn {
+  lazy val bitcoindRpc = project in Paths.get("..", "bitcoind-rpc").toFile
+  bitcoindRpc
+}
+
+lazy val downloadEclair = taskKey[Unit] {
+  "Download Eclair binaries, extract ./binaries/eclair"
+}
+
+downloadEclair := {
+  val logger = streams.value.log
+  import scala.sys.process._
+
+  val binaryDir = Paths.get("binaries", "eclair")
+
+  if (Files.notExists(binaryDir)) {
+    logger.info(s"Creating directory for Eclair binaires: $binaryDir")
+    Files.createDirectories(binaryDir)
+  }
+
+  val version = "0.3.1"
+  val commit = "6906ecb"
+
+  logger.debug(s"(Maybe) downloading Eclair binaries for version: $version")
+
+  val versionDir = binaryDir resolve version
+  val location =
+    s"https://github.com/ACINQ/eclair/releases/download/v$version/eclair-node-$version-$commit.jar"
+
+  if (Files.exists(versionDir)) {
+    logger.debug(
+      s"Directory $versionDir already exists, skipping download of Eclair $version")
+  } else {
+    logger.info(s"Creating directory $version")
+    Files.createDirectories(versionDir)
+
+    val destination = versionDir resolve s"eclair-node-$version-$commit.jar"
+    logger.info(
+      s"Downloading Eclair $version from location: $location, to destination: $destination")
+    (url(location) #> destination.toFile).!!
+
+    logger.info(s"Download complete")
+  }
+}

eclair-rpc/src/main/scala/org/bitcoins/eclair/rpc/client/EclairRpcClient.scala

@@ -37,8 +37,13 @@ import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.sys.process._
 import scala.util.{Failure, Properties, Success}
+import java.nio.file.NoSuchFileException
 
-class EclairRpcClient(val instance: EclairInstance)(
+/**
+  * @param binary Path to Eclair Jar. If not present, reads
+  *               environment variable `ECLAIR_PATH`
+  */
+class EclairRpcClient(val instance: EclairInstance, binary: Option[File] = None)(
     implicit system: ActorSystem)
     extends EclairApi {
   import JsonReaders._
@@ -633,21 +638,33 @@ class EclairRpcClient(val instance: EclairInstance)(
   }
 
   private def pathToEclairJar: String = {
-    val path = Properties
-      .envOrNone("ECLAIR_PATH")
-      .getOrElse(throw new RuntimeException(
-        List("Environment variable ECLAIR_PATH is not set!",
-             "This needs to be set to the directory containing the Eclair Jar")
-          .mkString(" ")))
 
-    val eclairV = "/eclair-node-0.3.1-6906ecb.jar"
-    val fullPath = path + eclairV
+    (binary, Properties.envOrNone("ECLAIR_PATH")) match {
+      // default to provided binary
+      case (Some(binary), _) =>
+        if (binary.exists) {
+          binary.toString
+        } else {
+          throw new NoSuchFileException(
+            s"Given binary ($binary) does not exist!")
+        }
+      case (None, Some(path)) =>
+        val eclairV =
+          s"/eclair-node-${EclairRpcClient.version}-${EclairRpcClient.commit}.jar"
+        val fullPath = path + eclairV
 
-    val jar = new File(fullPath)
-    if (jar.exists) {
-      fullPath
-    } else {
-      throw new RuntimeException(s"Could not Eclair Jar at location $fullPath")
+        val jar = new File(fullPath)
+        if (jar.exists) {
+          fullPath
+        } else {
+          throw new NoSuchFileException(
+            s"Could not Eclair Jar at location $fullPath")
+        }
+      case (None, None) =>
+        val msg = List(
+          "Environment variable ECLAIR_PATH is not set, and no binary is given!",
+          "Either needs to be set in order to start Eclair.")
+        throw new RuntimeException(msg.mkString(" "))
     }
   }
 
@@ -774,3 +791,12 @@ class EclairRpcClient(val instance: EclairInstance)(
     f
   }
 }
+
+object EclairRpcClient {
+
+  /** The current commit we support of Eclair */
+  private[bitcoins] val commit = "6906ecb"
+
+  /** The current version we support of Eclair */
+  private[bitcoins] val version = "0.3.1"
+}

testkit/src/main/scala/org/bitcoins/testkit/eclair/rpc/EclairRpcTestUtil.scala

@@ -29,6 +29,9 @@ import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.{Failure, Success}
 import org.bitcoins.rpc.config.BitcoindAuthCredentials
+import java.nio.file.Paths
+import scala.util.Properties
+import java.nio.file.Files
 
 /**
   * @define nodeLinkDoc
@@ -44,6 +47,32 @@ import org.bitcoins.rpc.config.BitcoindAuthCredentials
 trait EclairRpcTestUtil extends BitcoinSLogger {
   import org.bitcoins.core.compat.JavaConverters._
 
+  /** Directory where sbt downloads Eclair binaries */
+  private[bitcoins] lazy val binaryDirectory = {
+    val baseDirectory = {
+      val cwd = Paths.get(Properties.userDir)
+      if (cwd.endsWith("eclair-rpc-test") || cwd.endsWith("bitcoind-rpc-test")) {
+        cwd.getParent()
+      } else cwd
+    }
+
+    baseDirectory.resolve("binaries").resolve("eclair")
+  }
+
+  /** Path to Jar downloaded by Eclair, if it exists */
+  private[bitcoins] lazy val binary: Option[File] = {
+    val path = binaryDirectory
+      .resolve(EclairRpcClient.version)
+      .resolve(
+        s"eclair-node-${EclairRpcClient.version}-${EclairRpcClient.commit}.jar")
+
+    if (Files.exists(path)) {
+      Some(path.toFile)
+    } else {
+      None
+    }
+  }
+
   def randomDirName: String =
     0.until(5).map(_ => scala.util.Random.alphanumeric.head).mkString
 
@@ -169,7 +198,7 @@ trait EclairRpcTestUtil extends BitcoinSLogger {
     }
 
     val randInstanceF = bitcoindRpcF.map(randomEclairInstance(_))
-    val eclairRpcF = randInstanceF.map(i => new EclairRpcClient(i))
+    val eclairRpcF = randInstanceF.map(i => new EclairRpcClient(i, binary))
 
     val startedF = eclairRpcF.flatMap(_.start())
 
@@ -179,7 +208,7 @@ trait EclairRpcTestUtil extends BitcoinSLogger {
   def cannonicalEclairClient()(
       implicit system: ActorSystem): EclairRpcClient = {
     val inst = cannonicalEclairInstance()
-    new EclairRpcClient(inst)
+    new EclairRpcClient(inst, binary)
   }
 
   def deleteTmpDir(dir: File): Boolean = {
@@ -441,13 +470,13 @@ trait EclairRpcTestUtil extends BitcoinSLogger {
       bitcoindRpcClientF.map(EclairRpcTestUtil.eclairInstance(_))
 
     val clientF = e1InstanceF.flatMap { e1 =>
-      val e = new EclairRpcClient(e1)
+      val e = new EclairRpcClient(e1, binary)
       logger.debug(
         s"Temp eclair directory created ${e.getDaemon.authCredentials.datadir}")
       e.start().map(_ => e)
     }
     val otherClientF = e2InstanceF.flatMap { e2 =>
-      val e = new EclairRpcClient(e2)
+      val e = new EclairRpcClient(e2, binary)
       logger.debug(
         s"Temp eclair directory created ${e.getDaemon.authCredentials.datadir}")
       e.start().map(_ => e)

testkit/src/main/scala/org/bitcoins/testkit/rpc/BitcoindRpcTestUtil.scala

@@ -46,6 +46,7 @@ import org.bitcoins.util.ListUtil
 import scala.annotation.tailrec
 import scala.collection.immutable.Map
 import scala.collection.mutable
+import scala.collection.JavaConverters._
 import scala.concurrent._
 import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.util._
@@ -54,6 +55,10 @@ import java.io.File
 import com.typesafe.config.Config
 import com.typesafe.config.ConfigFactory
 import java.nio.file.Path
+import org.bitcoins.rpc.client.common.BitcoindVersion.Unknown
+import org.bitcoins.rpc.client.common.BitcoindVersion.V16
+import org.bitcoins.rpc.client.common.BitcoindVersion.V17
+import java.nio.file.Files
 
 //noinspection AccessorLikeMethodIsEmptyParen
 trait BitcoindRpcTestUtil extends BitcoinSLogger {
@@ -141,33 +146,46 @@ trait BitcoindRpcTestUtil extends BitcoinSLogger {
 
   lazy val network: RegTest.type = RegTest
 
-  private val V16_ENV = "BITCOIND_V16_PATH"
-  private val V17_ENV = "BITCOIND_V17_PATH"
-
-  private def getFileFromEnv(env: String): File = {
-    val envValue = Properties
-      .envOrNone(env)
-      .getOrElse(
-        throw new IllegalArgumentException(
-          s"$env environment variable is not set"))
-
-    val maybeDir = new File(envValue.trim)
-
-    val binary = if (maybeDir.isDirectory) {
-      Paths.get(maybeDir.getAbsolutePath, "bitcoind").toFile
-    } else {
-      maybeDir
+  /** The directory that sbt downloads bitcoind binaries into */
+  private[bitcoins] val binaryDirectory = {
+    val baseDirectory = {
+      val cwd = Paths.get(Properties.userDir)
+      if (cwd.endsWith("bitcoind-rpc-test") || cwd.endsWith("eclair-rpc-test")) {
+        cwd.getParent()
+      } else cwd
     }
 
-    binary
+    baseDirectory.resolve("binaries").resolve("bitcoind")
   }
 
-  private def getBinary(version: BitcoindVersion): File =
-    version match {
-      case BitcoindVersion.V16     => getFileFromEnv(V16_ENV)
-      case BitcoindVersion.V17     => getFileFromEnv(V17_ENV)
-      case BitcoindVersion.Unknown => BitcoindInstance.DEFAULT_BITCOIND_LOCATION
-    }
+  private def getBinary(version: BitcoindVersion): File = version match {
+    // default to newest version
+    case Unknown => getBinary(BitcoindVersion.newest)
+    case known @ (V16 | V17) =>
+      val versionFolder = Files
+        .list(binaryDirectory)
+        .iterator()
+        .asScala
+        .toList
+        .filter { f =>
+          val isFolder = Files.isDirectory(f)
+          val matchesVersion = f.toString.contains {
+            // drop leading 'v'
+            known.toString.drop(1)
+          }
+          isFolder && matchesVersion
+        }
+        // might be multiple versions downloaded for
+        // each major version, i.e. 0.16.2 and 0.16.3
+        .sorted
+        // we want the most recent one
+        .last
+
+      versionFolder
+        .resolve("bin")
+        .resolve("bitcoind")
+        .toFile()
+  }
 
   /** Creates a `bitcoind` instance within the user temporary directory */
   def instance(
@@ -181,10 +199,26 @@ trait BitcoindRpcTestUtil extends BitcoinSLogger {
     val configFile = writtenConfig(uri, rpcUri, zmqPort, pruneMode)
     val conf = BitcoindConfig(configFile)
     val auth = BitcoindAuthCredentials.fromConfig(conf)
-    val binary = versionOpt match {
-      case Some(version) =>
-        getBinary(version)
-      case None => BitcoindInstance.DEFAULT_BITCOIND_LOCATION
+    val binary: File = versionOpt match {
+      case Some(version) => getBinary(version)
+      case None =>
+        Try {
+          BitcoindInstance.DEFAULT_BITCOIND_LOCATION
+        }.recoverWith {
+          case _: RuntimeException =>
+            if (Files.exists(
+                  BitcoindRpcTestUtil.binaryDirectory
+                )) {
+              Success(getBinary(BitcoindVersion.newest))
+            } else {
+              Failure(new RuntimeException(
+                "Could not locate bitcoind. Make sure it is installed on your PATH, or if working with Bitcoin-S directly, try running 'sbt downloadBitcoind'"))
+            }
+
+        } match {
+          case Failure(exception) => throw exception
+          case Success(value)     => value
+        }
     }
     val instance = BitcoindInstance(network = network,
                                     uri = uri,

testkit/src/main/scala/org/bitcoins/testkit/util/BitcoindRpcTest.scala

@@ -10,8 +10,28 @@ import org.slf4j.{Logger, LoggerFactory}
 import scala.collection.mutable
 import scala.concurrent.duration.DurationInt
 import scala.concurrent.{Await, ExecutionContext}
+import java.nio.file.Files
 
 abstract class BitcoindRpcTest extends AsyncFlatSpec with BeforeAndAfterAll {
+
+  private val dirExists = Files.exists(BitcoindRpcTestUtil.binaryDirectory)
+  private val hasContents = dirExists && Files
+    .list(BitcoindRpcTestUtil.binaryDirectory)
+    .toArray()
+    .nonEmpty
+
+  if (!hasContents) {
+    import System.err.{println => printerr}
+    printerr()
+    printerr(s"Run 'sbt downloadBitcoind' to fetch needed binaries")
+    sys.error {
+      val msg =
+        s""""bitcoind binary directory (${BitcoindRpcTestUtil.binaryDirectory}) is empty. 
+        |Run 'sbt downloadBitcoind' to fetch needed binaries""".stripMargin
+      msg
+    }
+  }
+
   protected val logger: Logger = LoggerFactory.getLogger(getClass)
 
   implicit val system: ActorSystem =