Merge branch 'master' into peer-ratelimit

TOR.md

@@ -0,0 +1,135 @@
+## How to Use Tor with Eclair
+
+### Installing Tor on your node
+
+#### Linux:
+
+```shell
+sudo apt install tor
+```
+
+#### Mac OS X:
+
+```shell
+brew install tor
+```
+
+#### Windows:
+  
+[Download the "Expert Bundle"](https://www.torproject.org/download/download.html) from Tor's website and extract it to `C:\tor`.
+
+### Configuring Tor
+
+#### Linux and Max OS X:
+
+Eclair requires safe cookie authentication as well as SOCKS5 and control connections to be enabled.
+
+Edit Tor configuration file `/etc/tor/torrc` (Linux) or `/usr/local/etc/tor/torrc` (Mac OS X).
+
+```
+SOCKSPort 9050
+ControlPort 9051
+CookieAuthentication 1
+ExitPolicy reject *:* # don't change this unless you really know what you are doing
+```
+
+Make sure eclair is allowed to read Tor's cookie file (typically `/var/run/tor/control.authcookie`).
+
+#### Windows:
+
+On Windows it is easier to use the password authentication mechanism.
+
+First pick a password and hash it with this command:
+
+```shell
+$ cd c:\tor\Tor
+$ tor --hash-password this-is-an-example-password-change-it
+16:94A50709CAA98333602756426F43E6AC6760B9ADEF217F58219E639E5A
+```
+
+Create a Tor configuration file (`C:\tor\Conf\torrc`), edit it and replace the value for `HashedControlPassword` with the result of the command above.
+
+```
+SOCKSPort 9050
+ControlPort 9051
+HashedControlPassword 16:--REPLACE--THIS--WITH--THE--HASH--OF--YOUR--PASSWORD--
+ExitPolicy reject *:* # don't change this unless you really know what you are doing
+```
+
+### Start Tor
+
+#### Linux:
+
+```shell
+sudo systemctl start tor
+```
+
+#### Mac OS X:
+
+```shell
+brew services start tor
+```
+
+#### Windows:
+
+Open a CMD with administrator access
+
+```shell
+cd c:\tor\Tor
+tor --service install -options -f c:\tor\Conf\torrc
+```
+
+### Configure Tor hidden service
+
+To create a Tor hidden service endpoint simply set the `eclair.tor.enabled` parameter in `eclair.conf` to true.
+```
+eclair.tor.enabled = true
+```
+Eclair will automatically set up a hidden service endpoint and add its onion address to the `server.public-ips` list.
+You can see what onion address is assigned using `eclair-cli`:
+
+```shell
+eclair-cli getinfo
+```
+Eclair saves the Tor endpoint's private key in `~/.eclair/tor_pk`, so that it can recreate the endpoint address after 
+restart. If you remove the private key eclair will regenerate the endpoint address.   
+
+There are two possible values for `protocol-version`:
+
+```
+eclair.tor.protocol-version = "v3"
+```
+
+value   | description
+--------|---------------------------------------------------------
+ v2     | set up a Tor hidden service version 2 end point
+ v3     | set up a Tor hidden service version 3 end point (default)
+ 
+Tor protocol v3 (supported by Tor version 0.3.3.6 and higher) is backwards compatible and supports 
+both v2 and v3 addresses. 
+
+For increased privacy do not advertise your IP address in the `server.public-ips` list, and set your binding IP to `localhost`:
+```
+eclair.server.binding-ip = "127.0.0.1"
+```
+
+### Configure SOCKS5 proxy
+
+By default all incoming connections will be established via Tor network, but all outgoing will be created via the 
+clearnet. To route them through Tor you can use Tor's SOCKS5 proxy. Add this line in your `eclair.conf`:
+```
+eclair.socks5.enabled = true
+```
+You can use SOCKS5 proxy only for specific types of addresses. Use `eclair.socks5.use-for-ipv4`, `eclair.socks5.use-for-ipv6`
+or `eclair.socks5.use-for-tor` for fine tuning.
+
+To create a new Tor circuit for every connection, use `randomize-credentials` parameter:
+
+```
+eclair.socks5.randomize-credentials = true
+```
+
+:warning: Tor hidden service and SOCKS5 are independent options. You can use just one of them, but if you want to get the most privacy 
+features from using Tor use both.  
+
+Note, that bitcoind should be configured to use Tor as well (https://en.bitcoin.it/wiki/Setting_up_a_Tor_hidden_service).

eclair-core/pom.xml

@@ -183,6 +183,11 @@
             <artifactId>scodec-core_${scala.version.short}</artifactId>
             <version>1.10.3</version>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.9</version>
+        </dependency>
         <!-- LOGGING -->
         <dependency>
             <groupId>org.clapper</groupId>
@@ -217,6 +222,12 @@
             <version>4.0.3</version>
         </dependency>
         <!-- TESTS -->
+        <dependency>
+            <groupId>com.softwaremill.quicklens</groupId>
+            <artifactId>quicklens_${scala.version.short}</artifactId>
+            <version>1.4.11</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>com.whisk</groupId>
             <artifactId>docker-testkit-scalatest_${scala.version.short}</artifactId>

eclair-core/src/main/resources/reference.conf

@@ -62,7 +62,7 @@ eclair {
   max-reserve-to-funding-ratio = 0.05 // channel reserve can't be more than 5% of the funding amount (recommended: 1%)
 
   to-remote-delay-blocks = 144 // number of blocks that the other node's to-self outputs must be delayed (144 ~ 1 day)
-  max-to-local-delay-blocks = 2000 // maximum number of blocks that we are ready to accept for our own delayed outputs (2000 ~ 2 weeks)
+  max-to-local-delay-blocks = 2016 // maximum number of blocks that we are ready to accept for our own delayed outputs (2016 ~ 2 weeks)
   mindepth-blocks = 3
   expiry-delta-blocks = 144
 
@@ -79,11 +79,6 @@ eclair {
 
   revocation-timeout = 20 seconds // after sending a commit_sig, we will wait for at most that duration before disconnecting
 
-  channel-exclude-duration = 60 seconds // when a temporary channel failure is returned, we exclude the channel from our payment routes for this duration
-  router-broadcast-interval = 60 seconds // see BOLT #7
-
-  router-init-timeout = 5 minutes
-
   ping-interval = 30 seconds
   ping-timeout = 10 seconds // will disconnect if peer takes longer than that to respond
   ping-disconnect = true // disconnect if no answer to our pings
@@ -96,4 +91,31 @@ eclair {
   min-funding-satoshis = 100000
 
   autoprobe-count = 0 // number of parallel tasks that send test payments to detect invalid channels
+
+  router {
+    randomize-route-selection = true // when computing a route for a payment we randomize the final selection
+    channel-exclude-duration = 60 seconds // when a temporary channel failure is returned, we exclude the channel from our payment routes for this duration
+    broadcast-interval = 60 seconds // see BOLT #7
+    init-timeout = 5 minutes
+  }
+
+  socks5 {
+    enabled = false
+    host = "127.0.0.1"
+    port = 9050
+    use-for-ipv4 = true
+    use-for-ipv6 = true
+    use-for-tor = true
+    randomize-credentials = false // this allows tor stream isolation
+  }
+
+  tor {
+    enabled = false
+    protocol = "v3" // v2, v3
+    auth = "password" // safecookie, password
+    password = "foobar" // used when auth=password
+    host = "127.0.0.1"
+    port = 9051
+    private-key-file = "tor.dat"
+  }
 }

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit
 
 import com.codahale.metrics.MetricRegistry
 import com.google.common.net.InetAddresses
+import com.google.common.net.{HostAndPort, InetAddresses}
 import com.typesafe.config.{Config, ConfigFactory}
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.{BinaryData, Block}
@@ -32,7 +33,8 @@ import fr.acinq.eclair.channel.Channel
 import fr.acinq.eclair.crypto.KeyManager
 import fr.acinq.eclair.db._
 import fr.acinq.eclair.db.sqlite._
-import fr.acinq.eclair.wire.Color
+import fr.acinq.eclair.tor.Socks5ProxyParams
+import fr.acinq.eclair.wire.{Color, NodeAddress}
 
 import scala.collection.JavaConversions._
 import scala.concurrent.duration.FiniteDuration
@@ -44,7 +46,7 @@ case class NodeParams(metrics: MetricRegistry = new MetricRegistry(),
                       keyManager: KeyManager,
                       alias: String,
                       color: Color,
-                      publicAddresses: List[InetSocketAddress],
+                      publicAddresses: List[NodeAddress],
                       globalFeatures: BinaryData,
                       localFeatures: BinaryData,
                       overrideFeatures: Map[PublicKey, (BinaryData, BinaryData)],
@@ -82,7 +84,9 @@ case class NodeParams(metrics: MetricRegistry = new MetricRegistry(),
                       paymentRequestExpiry: FiniteDuration,
                       maxPendingPaymentRequests: Int,
                       maxPaymentFee: Double,
-                      minFundingSatoshis: Long) {
+                      minFundingSatoshis: Long,
+                      randomizeRouteSelection: Boolean,
+                      socksProxy_opt: Option[Socks5ProxyParams]) {
 
   val privateKey = keyManager.nodeKey.privateKey
   val nodeId = keyManager.nodeId
@@ -130,7 +134,7 @@ object NodeParams {
     }
   }
 
-  def makeNodeParams(datadir: File, config: Config, keyManager: KeyManager): NodeParams = {
+  def makeNodeParams(datadir: File, config: Config, keyManager: KeyManager, torAddress_opt: Option[NodeAddress]): NodeParams = {
 
     datadir.mkdirs()
 
@@ -183,11 +187,28 @@ object NodeParams {
       (p -> (gf, lf))
     }.toMap
 
+    val socksProxy_opt = if (config.getBoolean("socks5.enabled")) {
+      Some(Socks5ProxyParams(
+        address = new InetSocketAddress(config.getString("socks5.host"), config.getInt("socks5.port")),
+        credentials_opt = None,
+        randomizeCredentials = config.getBoolean("socks5.randomize-credentials"),
+        useForIPv4 = config.getBoolean("socks5.use-for-ipv4"),
+        useForIPv6 = config.getBoolean("socks5.use-for-ipv6"),
+        useForTor = config.getBoolean("socks5.use-for-tor")
+      ))
+    } else {
+      None
+    }
+
+    val addresses = config.getStringList("server.public-ips")
+      .toList
+      .map(ip => NodeAddress.fromParts(ip, config.getInt("server.port")).get) ++ torAddress_opt
+
     NodeParams(
       keyManager = keyManager,
       alias = nodeAlias,
       color = Color(color.data(0), color.data(1), color.data(2)),
-      publicAddresses = config.getStringList("server.public-ips").toList.map(ip => new InetSocketAddress(InetAddresses.forString(ip), config.getInt("server.port"))),
+      publicAddresses = addresses,
       globalFeatures = BinaryData(config.getString("global-features")),
       localFeatures = BinaryData(config.getString("local-features")),
       overrideFeatures = overrideFeatures,
@@ -211,7 +232,7 @@ object NodeParams {
       paymentsDb = paymentsDb,
       auditDb = auditDb,
       revocationTimeout = FiniteDuration(config.getDuration("revocation-timeout").getSeconds, TimeUnit.SECONDS),
-      routerBroadcastInterval = FiniteDuration(config.getDuration("router-broadcast-interval").getSeconds, TimeUnit.SECONDS),
+      routerBroadcastInterval = FiniteDuration(config.getDuration("router.broadcast-interval").getSeconds, TimeUnit.SECONDS),
       pingInterval = FiniteDuration(config.getDuration("ping-interval").getSeconds, TimeUnit.SECONDS),
       pingTimeout = FiniteDuration(config.getDuration("ping-timeout").getSeconds, TimeUnit.SECONDS),
       pingDisconnect = config.getBoolean("ping-disconnect"),
@@ -220,12 +241,14 @@ object NodeParams {
       autoReconnect = config.getBoolean("auto-reconnect"),
       chainHash = chainHash,
       channelFlags = config.getInt("channel-flags").toByte,
-      channelExcludeDuration = FiniteDuration(config.getDuration("channel-exclude-duration").getSeconds, TimeUnit.SECONDS),
+      channelExcludeDuration = FiniteDuration(config.getDuration("router.channel-exclude-duration").getSeconds, TimeUnit.SECONDS),
       watcherType = watcherType,
       paymentRequestExpiry = FiniteDuration(config.getDuration("payment-request-expiry").getSeconds, TimeUnit.SECONDS),
       maxPendingPaymentRequests = config.getInt("max-pending-payment-requests"),
       maxPaymentFee = config.getDouble("max-payment-fee"),
-      minFundingSatoshis = config.getLong("min-funding-satoshis")
+      minFundingSatoshis = config.getLong("min-funding-satoshis"),
+      randomizeRouteSelection = config.getBoolean("router.randomize-route-selection"),
+      socksProxy_opt = socksProxy_opt
     )
   }
 }

eclair-core/src/main/scala/fr/acinq/eclair/PortChecker.scala

@@ -16,7 +16,7 @@
 
 package fr.acinq.eclair
 
-import java.net.{InetAddress, ServerSocket}
+import java.net.{InetAddress, InetSocketAddress, ServerSocket}
 
 import scala.util.{Failure, Success, Try}
 
@@ -28,8 +28,12 @@ object PortChecker {
     *
     * @return
     */
-  def checkAvailable(host: String, port: Int): Unit = {
+  def checkAvailable(host: String, port: Int): Unit = checkAvailable(InetAddress.getByName(host), port)
-    Try(new ServerSocket(port, 50, InetAddress.getByName(host))) match {
+
+  def checkAvailable(socketAddress: InetSocketAddress): Unit = checkAvailable(socketAddress.getAddress, socketAddress.getPort)
+
+  def checkAvailable(address: InetAddress, port: Int): Unit = {
+    Try(new ServerSocket(port, 50, address)) match {
       case Success(socket) =>
         Try(socket.close())
       case Failure(_) =>

eclair-core/src/main/scala/fr/acinq/eclair/Setup.scala

@@ -21,9 +21,11 @@ import org.slf4j.LoggerFactory
 import java.util.concurrent.TimeUnit
 import java.io.File
 import java.net.InetSocketAddress
+import java.nio.file.Paths
 import java.sql.DriverManager
 import java.util.concurrent.TimeUnit
 
+import akka.Done
 import akka.actor.{ActorRef, ActorSystem, Props, SupervisorStrategy}
 import akka.http.scaladsl.Http
 import akka.pattern.after
@@ -48,11 +50,14 @@ import fr.acinq.eclair.crypto.LocalKeyManager
 import fr.acinq.eclair.io.{Authenticator, Server, Switchboard}
 import fr.acinq.eclair.payment._
 import fr.acinq.eclair.router._
+import fr.acinq.eclair.tor.TorProtocolHandler.OnionServiceVersion
+import fr.acinq.eclair.tor.{Controller, TorProtocolHandler}
+import fr.acinq.eclair.wire.NodeAddress
 import grizzled.slf4j.Logging
 import org.json4s.JsonAST.JArray
 
-import scala.concurrent.duration._
 import scala.concurrent._
+import scala.concurrent.duration._
 
 /**
   * Setup eclair from a data directory.
@@ -70,26 +75,34 @@ class Setup(datadir: File,
   logger.info(s"hello!")
   logger.info(s"version=${getClass.getPackage.getImplementationVersion} commit=${getClass.getPackage.getSpecificationVersion}")
   logger.info(s"datadir=${datadir.getCanonicalPath}")
-
+  logger.info(s"initializing secure random generator")
+  // this will force the secure random instance to initialize itself right now, making sure it doesn't hang later (see comment in package.scala)
+  secureRandom.nextInt()
 
   val config = NodeParams.loadConfiguration(datadir, overrideDefaults)
   val seed = seed_opt.getOrElse(NodeParams.getSeed(datadir))
   val chain = config.getString("chain")
   val keyManager = new LocalKeyManager(seed, NodeParams.makeChainHash(chain))
-  val nodeParams = NodeParams.makeNodeParams(datadir, config, keyManager)
+  implicit val materializer = ActorMaterializer()
+  implicit val timeout = Timeout(30 seconds)
+  implicit val formats = org.json4s.DefaultFormats
+  implicit val ec = ExecutionContext.Implicits.global
+  implicit val sttpBackend = OkHttpFutureBackend()
+
+  val nodeParams = NodeParams.makeNodeParams(datadir, config, keyManager, initTor())
+
+  val serverBindingAddress = new InetSocketAddress(
+    config.getString("server.binding-ip"),
+    config.getInt("server.port"))
 
   // early checks
   DBCompatChecker.checkDBCompatibility(nodeParams)
   DBCompatChecker.checkNetworkDBCompatibility(nodeParams)
-  PortChecker.checkAvailable(config.getString("server.binding-ip"), config.getInt("server.port"))
+  PortChecker.checkAvailable(serverBindingAddress)
 
   logger.info(s"nodeid=${nodeParams.nodeId} alias=${nodeParams.alias}")
   logger.info(s"using chain=$chain chainHash=${nodeParams.chainHash}")
 
-  logger.info(s"initializing secure random generator")
-  // this will force the secure random instance to initialize itself right now, making sure it doesn't hang later (see comment in package.scala)
-  secureRandom.nextInt()
-
   Slf4jReporter
     .forRegistry(nodeParams.metrics)
     .outputTo(LoggerFactory.getLogger("fr.acinq.eclair.metrics"))
@@ -117,9 +130,6 @@ class Setup(datadir: File,
     .build
     .start(1, TimeUnit.HOURS)
 
-  implicit val ec = ExecutionContext.Implicits.global
-  implicit val sttpBackend = OkHttpFutureBackend()
-
   val bitcoin = nodeParams.watcherType match {
     case BITCOIND =>
       val bitcoinClient = new BasicBitcoinJsonRPCClient(
@@ -185,11 +195,11 @@ class Setup(datadir: File,
   def bootstrap: Future[Kit] = {
     for {
       _ <- Future.successful(true)
-      feeratesRetrieved = Promise[Boolean]()
+      feeratesRetrieved = Promise[Done]()
-      zmqBlockConnected = Promise[Boolean]()
+      zmqBlockConnected = Promise[Done]()
-      zmqTxConnected = Promise[Boolean]()
+      zmqTxConnected = Promise[Done]()
-      tcpBound = Promise[Unit]()
+      tcpBound = Promise[Done]()
-      routerInitialized = Promise[Unit]()
+      routerInitialized = Promise[Done]()
 
       defaultFeerates = FeeratesPerKB(
         block_1 = config.getLong("default-feerates.delay-blocks.1"),
@@ -214,7 +224,7 @@ class Setup(datadir: File,
           Globals.feeratesPerKw.set(FeeratesPerKw(feerates))
           system.eventStream.publish(CurrentFeerates(Globals.feeratesPerKw.get))
           logger.info(s"current feeratesPerKB=${Globals.feeratesPerKB.get()} feeratesPerKw=${Globals.feeratesPerKw.get()}")
-          feeratesRetrieved.trySuccess(true)
+          feeratesRetrieved.trySuccess(Done)
       })
       _ <- feeratesRetrieved.future
 
@@ -224,13 +234,13 @@ class Setup(datadir: File,
           system.actorOf(SimpleSupervisor.props(Props(new ZMQActor(config.getString("bitcoind.zmqtx"), Some(zmqTxConnected))), "zmqtx", SupervisorStrategy.Restart))
           system.actorOf(SimpleSupervisor.props(ZmqWatcher.props(new ExtendedBitcoinClient(new BatchingBitcoinJsonRPCClient(bitcoinClient))), "watcher", SupervisorStrategy.Resume))
         case Electrum(electrumClient) =>
-          zmqBlockConnected.success(true)
+          zmqBlockConnected.success(Done)
-          zmqTxConnected.success(true)
+          zmqTxConnected.success(Done)
           system.actorOf(SimpleSupervisor.props(Props(new ElectrumWatcher(electrumClient)), "watcher", SupervisorStrategy.Resume))
       }
 
       router = system.actorOf(SimpleSupervisor.props(Router.props(nodeParams, watcher, Some(routerInitialized)), "router", SupervisorStrategy.Resume))
-      routerTimeout = after(FiniteDuration(config.getDuration("router-init-timeout").getSeconds, TimeUnit.SECONDS), using = system.scheduler)(Future.failed(new RuntimeException("Router initialization timed out")))
+      routerTimeout = after(FiniteDuration(config.getDuration("router.init-timeout").getSeconds, TimeUnit.SECONDS), using = system.scheduler)(Future.failed(new RuntimeException("Router initialization timed out")))
       _ <- Future.firstCompletedOf(routerInitialized.future :: routerTimeout :: Nil)
 
       wallet = bitcoin match {
@@ -257,7 +267,7 @@ class Setup(datadir: File,
       relayer = system.actorOf(SimpleSupervisor.props(Relayer.props(nodeParams, register, paymentHandler), "relayer", SupervisorStrategy.Resume))
       authenticator = system.actorOf(SimpleSupervisor.props(Authenticator.props(nodeParams), "authenticator", SupervisorStrategy.Resume))
       switchboard = system.actorOf(SimpleSupervisor.props(Switchboard.props(nodeParams, authenticator, watcher, router, relayer, wallet), "switchboard", SupervisorStrategy.Resume))
-      server = system.actorOf(SimpleSupervisor.props(Server.props(nodeParams, authenticator, new InetSocketAddress(config.getString("server.binding-ip"), config.getInt("server.port")), Some(tcpBound)), "server", SupervisorStrategy.Restart))
+      server = system.actorOf(SimpleSupervisor.props(Server.props(nodeParams, authenticator, serverBindingAddress, Some(tcpBound)), "server", SupervisorStrategy.Restart))
       paymentInitiator = system.actorOf(SimpleSupervisor.props(PaymentInitiator.props(nodeParams.nodeId, router, register), "payment-initiator", SupervisorStrategy.Restart))
       _ = for (i <- 0 until config.getInt("autoprobe-count")) yield system.actorOf(SimpleSupervisor.props(Autoprobe.props(nodeParams, router, paymentInitiator), s"payment-autoprobe-$i", SupervisorStrategy.Restart))
 
@@ -298,7 +308,8 @@ class Setup(datadir: File,
               alias = nodeParams.alias,
               port = config.getInt("server.port"),
               chainHash = nodeParams.chainHash,
-              blockHeight = Globals.blockCount.intValue()))
+              blockHeight = Globals.blockCount.intValue(),
+              publicAddresses = nodeParams.publicAddresses))
 
           override def appKit: Kit = kit
 
@@ -324,6 +335,31 @@ class Setup(datadir: File,
       throw e
   }
 
+  private def initTor(): Option[NodeAddress] = {
+    if (config.getBoolean("tor.enabled")) {
+      val promiseTorAddress = Promise[NodeAddress]()
+      val auth = config.getString("tor.auth") match {
+        case "password" => TorProtocolHandler.Password(config.getString("tor.password"))
+        case "safecookie" => TorProtocolHandler.SafeCookie()
+      }
+      val protocolHandlerProps = TorProtocolHandler.props(
+        version = OnionServiceVersion(config.getString("tor.protocol")),
+        authentication = auth,
+        privateKeyPath = new File(datadir, config.getString("tor.private-key-file")).toPath,
+        virtualPort = config.getInt("server.port"),
+        onionAdded = Some(promiseTorAddress))
+
+      val controller = system.actorOf(SimpleSupervisor.props(Controller.props(
+        address = new InetSocketAddress(config.getString("tor.host"), config.getInt("tor.port")),
+        protocolHandlerProps = protocolHandlerProps), "tor", SupervisorStrategy.Stop))
+
+      val torAddress = await(promiseTorAddress.future, 30 seconds, "tor did not respond after 30 seconds")
+      logger.info(s"Tor address $torAddress")
+      Some(torAddress)
+    } else {
+      None
+    }
+  }
 }
 
 // @formatter:off

eclair-core/src/main/scala/fr/acinq/eclair/api/JsonSerializers.scala

@@ -123,10 +123,7 @@ class FailureMessageSerializer extends CustomSerializer[FailureMessage](format =
 }))
 
 class NodeAddressSerializer extends CustomSerializer[NodeAddress](format => ({ null},{
-  case IPv4(a, p) => JString(HostAndPort.fromParts(a.getHostAddress, p).toString)
+  case n: NodeAddress => JString(HostAndPort.fromParts(n.socketAddress.getHostString, n.socketAddress.getPort).toString)
-  case IPv6(a, p) => JString(HostAndPort.fromParts(a.getHostAddress, p).toString)
-  case Tor2(b, p) => JString(s"${b.toString}:$p")
-  case Tor3(b, p) => JString(s"${b.toString}:$p")
 }))
 
 class DirectionSerializer extends CustomSerializer[Direction](format => ({ null },{

eclair-core/src/main/scala/fr/acinq/eclair/api/Service.scala

@@ -41,7 +41,7 @@ import fr.acinq.eclair.io.{NodeURI, Peer}
 import fr.acinq.eclair.payment.PaymentLifecycle._
 import fr.acinq.eclair.payment._
 import fr.acinq.eclair.router.{ChannelDesc, RouteRequest, RouteResponse, Router}
-import fr.acinq.eclair.wire.{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement}
+import fr.acinq.eclair.wire.{ChannelAnnouncement, ChannelUpdate, NodeAddress, NodeAnnouncement}
 import fr.acinq.eclair.{Kit, ShortChannelId, feerateByte2Kw}
 import grizzled.slf4j.Logging
 import org.json4s.JsonAST.{JBool, JInt, JString}
@@ -56,7 +56,7 @@ case class JsonRPCBody(jsonrpc: String = "1.0", id: String = "eclair-node", meth
 case class Error(code: Int, message: String)
 case class JsonRPCRes(result: AnyRef, error: Option[Error], id: String)
 case class Status(node_id: String)
-case class GetInfoResponse(nodeId: PublicKey, alias: String, port: Int, chainHash: BinaryData, blockHeight: Int)
+case class GetInfoResponse(nodeId: PublicKey, alias: String, port: Int, chainHash: BinaryData, blockHeight: Int, publicAddresses: Seq[NodeAddress])
 case class AuditResponse(sent: Seq[PaymentSent], received: Seq[PaymentReceived], relayed: Seq[PaymentRelayed])
 trait RPCRejection extends Rejection {
   def requestId: String
@@ -184,8 +184,8 @@ trait Service extends Logging {
                         }
                         // local network methods
                         case "peers" => completeRpcFuture(req.id, for {
-                          peers <- (switchboard ? 'peers).mapTo[Map[PublicKey, ActorRef]]
+                          peers <- (switchboard ? 'peers).mapTo[Iterable[ActorRef]]
-                          peerinfos <- Future.sequence(peers.values.map(peer => (peer ? GetPeerInfo).mapTo[PeerInfo]))
+                          peerinfos <- Future.sequence(peers.map(peer => (peer ? GetPeerInfo).mapTo[PeerInfo]))
                         } yield peerinfos)
                         case "channels" => req.params match {
                           case Nil =>

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/WatcherTypes.scala

@@ -77,6 +77,11 @@ final case class WatchEventLost(event: BitcoinEvent) extends WatchEvent
   */
 final case class PublishAsap(tx: Transaction)
 final case class ValidateRequest(ann: ChannelAnnouncement)
-final case class ValidateResult(c: ChannelAnnouncement, tx: Option[Transaction], unspent: Boolean, t: Option[Throwable])
+sealed trait UtxoStatus
+object UtxoStatus {
+  case object Unspent extends UtxoStatus
+  case class Spent(spendingTxConfirmed: Boolean) extends UtxoStatus
+}
+final case class ValidateResult(c: ChannelAnnouncement, fundingTx: Either[Throwable, (Transaction, UtxoStatus)])
 
 // @formatter:on

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/rpc/ExtendedBitcoinClient.scala

@@ -19,7 +19,7 @@ package fr.acinq.eclair.blockchain.bitcoind.rpc
 import fr.acinq.bitcoin._
 import fr.acinq.eclair.ShortChannelId.coordinates
 import fr.acinq.eclair.TxCoordinates
-import fr.acinq.eclair.blockchain.ValidateResult
+import fr.acinq.eclair.blockchain.{UtxoStatus, ValidateResult}
 import fr.acinq.eclair.wire.ChannelAnnouncement
 import org.json4s.JsonAST._
 
@@ -153,8 +153,16 @@ class ExtendedBitcoinClient(val rpcClient: BitcoinJsonRPCClient) {
       }
       tx <- getRawTransaction(txid)
       unspent <- isTransactionOutputSpendable(txid, outputIndex, includeMempool = true)
-    } yield ValidateResult(c, Some(Transaction.read(tx)), unspent, None)
+      fundingTxStatus <- if (unspent) {
+        Future.successful(UtxoStatus.Unspent)
+      } else {
+        // if this returns true, it means that the spending tx is *not* in the blockchain
+        isTransactionOutputSpendable(txid, outputIndex, includeMempool = false).map {
+          case res => UtxoStatus.Spent(spendingTxConfirmed = !res)
+        }
+      }
+    } yield ValidateResult(c, Right((Transaction.read(tx), fundingTxStatus)))
 
-  } recover { case t: Throwable => ValidateResult(c, None, false, Some(t)) }
+  } recover { case t: Throwable => ValidateResult(c, Left(t)) }
 
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/bitcoind/zmq/ZMQActor.scala

@@ -16,6 +16,7 @@
 
 package fr.acinq.eclair.blockchain.bitcoind.zmq
 
+import akka.Done
 import akka.actor.{Actor, ActorLogging}
 import fr.acinq.bitcoin.{Block, Transaction}
 import fr.acinq.eclair.blockchain.{NewBlock, NewTransaction}
@@ -30,7 +31,7 @@ import scala.util.Try
 /**
   * Created by PM on 04/04/2017.
   */
-class ZMQActor(address: String, connected: Option[Promise[Boolean]] = None) extends Actor with ActorLogging {
+class ZMQActor(address: String, connected: Option[Promise[Done]] = None) extends Actor with ActorLogging {
 
   import ZMQActor._
 
@@ -79,7 +80,7 @@ class ZMQActor(address: String, connected: Option[Promise[Boolean]] = None) exte
     case event: Event => event.getEvent match {
       case ZMQ.EVENT_CONNECTED =>
         log.info(s"connected to ${event.getAddress}")
-        Try(connected.map(_.success(true)))
+        Try(connected.map(_.success(Done)))
         context.system.eventStream.publish(ZMQConnected)
       case ZMQ.EVENT_DISCONNECTED =>
         log.warning(s"disconnected from ${event.getAddress}")

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumClient.scala

@@ -90,14 +90,9 @@ class ElectrumClient(serverAddress: InetSocketAddress, ssl: SSL)(implicit val ec
 
   val channelOpenFuture = b.connect(serverAddress.getHostName, serverAddress.getPort)
 
-  def close() = {
-    statusListeners.map(_ ! ElectrumDisconnected)
-    context stop self
-  }
-
   def errorHandler(t: Throwable) = {
     log.info("server={} connection error (reason={})", serverAddress, t.getMessage)
-    close()
+    self ! Close
   }
 
   channelOpenFuture.addListeners(new ChannelFutureListener {
@@ -111,7 +106,7 @@ class ElectrumClient(serverAddress: InetSocketAddress, ssl: SSL)(implicit val ec
               errorHandler(future.cause())
             } else {
               log.info("server={} channel closed: {}", serverAddress, future.channel())
-              close()
+              self ! Close
             }
           }
         })
@@ -205,7 +200,6 @@ class ElectrumClient(serverAddress: InetSocketAddress, ssl: SSL)(implicit val ec
   val headerSubscriptions = collection.mutable.HashSet.empty[ActorRef]
   val version = ServerVersion(CLIENT_NAME, PROTOCOL_VERSION)
   val statusListeners = collection.mutable.HashSet.empty[ActorRef]
-  val keepHeaders = 100
 
   var reqId = 0
 
@@ -224,6 +218,10 @@ class ElectrumClient(serverAddress: InetSocketAddress, ssl: SSL)(implicit val ec
 
       case PingResponse => ()
 
+      case Close =>
+        statusListeners.map(_ ! ElectrumDisconnected)
+        context.stop(self)
+
       case _ => log.warning("server={} unhandled message {}", serverAddress, message)
     }
   }
@@ -273,7 +271,7 @@ class ElectrumClient(serverAddress: InetSocketAddress, ssl: SSL)(implicit val ec
           context become waitingForTip(ctx)
         case ServerError(request, error) =>
           log.error("server={} sent error={} while processing request={}, disconnecting", serverAddress, error, request)
-          close()
+          self ! Close
       }
 
     case AddStatusListener(actor) => statusListeners += actor
@@ -368,7 +366,7 @@ object ElectrumClient {
   case class GetAddressHistoryResponse(address: String, history: Seq[TransactionHistoryItem]) extends Response
 
   case class GetScriptHashHistory(scriptHash: BinaryData) extends Request
-  case class GetScriptHashHistoryResponse(scriptHash: BinaryData, history: Seq[TransactionHistoryItem]) extends Response
+  case class GetScriptHashHistoryResponse(scriptHash: BinaryData, history: List[TransactionHistoryItem]) extends Response
 
   case class AddressListUnspent(address: String) extends Request
   case class UnspentItem(tx_hash: BinaryData, tx_pos: Int, value: Long, height: Long) {
@@ -457,6 +455,8 @@ object ElectrumClient {
     case object LOOSE extends SSL
   }
 
+  case object Close
+
   // @formatter:on
 
   def parseResponse(input: String): Either[Response, JsonRPCResponse] = {

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWallet.scala

@@ -22,7 +22,7 @@ import fr.acinq.bitcoin.DeterministicWallet.{ExtendedPrivateKey, derivePrivateKe
 import fr.acinq.bitcoin.{Base58, Base58Check, BinaryData, Block, BlockHeader, Crypto, DeterministicWallet, OP_PUSHDATA, OutPoint, SIGHASH_ALL, Satoshi, Script, ScriptElt, ScriptWitness, SigVersion, Transaction, TxIn, TxOut}
 import fr.acinq.eclair.blockchain.bitcoind.rpc.Error
 import fr.acinq.eclair.blockchain.electrum.ElectrumClient._
-import fr.acinq.eclair.blockchain.electrum.db.WalletDb
+import fr.acinq.eclair.blockchain.electrum.db.{HeaderDb, WalletDb}
 import fr.acinq.eclair.transactions.Transactions
 import grizzled.slf4j.Logging
 
@@ -64,28 +64,39 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
   // +--------------------------------------------+
 
   /**
-    * Send a notification if the wallet is ready and its ready message has not
+    * If the wallet is ready and its state changed since the last time it was ready:
-    * already been sent
+    * - publish a `WalletReady` notification
+    * - persist state data
     *
     * @param data wallet data
     * @return the input data with an updated 'last ready message' if needed
     */
-  def notifyReady(data: ElectrumWallet.Data) : ElectrumWallet.Data = {
+  def persistAndNotify(data: ElectrumWallet.Data): ElectrumWallet.Data = {
     if (data.isReady(swipeRange)) {
       data.lastReadyMessage match {
         case Some(value) if value == data.readyMessage =>
           log.debug(s"ready message $value has already been sent")
           data
         case _ =>
+          log.info(s"checking wallet")
           val ready = data.readyMessage
           log.info(s"wallet is ready with $ready")
           context.system.eventStream.publish(ready)
           context.system.eventStream.publish(NewWalletReceiveAddress(data.currentReceiveAddress))
+          params.walletDb.persist(PersistentData(data))
           data.copy(lastReadyMessage = Some(ready))
       }
     } else data
   }
 
+  // sent notifications for all wallet transactions
+  def advertiseTransactions(data: ElectrumWallet.Data): Unit = {
+    data.transactions.values.foreach(tx => data.computeTransactionDelta(tx).foreach {
+      case (received, sent, fee_opt) =>
+        context.system.eventStream.publish(TransactionReceived(tx, data.computeTransactionDepth(tx.txid), received, sent, fee_opt, data.computeTimestamp(tx.txid, params.walletDb)))
+    })
+  }
+
   startWith(DISCONNECTED, {
     val blockchain = params.chainHash match {
       // regtest is a special case, there are no checkpoints and we start with a single header
@@ -97,13 +108,33 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
     val headers = params.walletDb.getHeaders(blockchain.checkpoints.size * RETARGETING_PERIOD, None)
     log.info(s"loading ${headers.size} headers from db")
     val blockchain1 = Blockchain.addHeadersChunk(blockchain, blockchain.checkpoints.size * RETARGETING_PERIOD, headers)
+    val data = Try(params.walletDb.readPersistentData()) match {
+      case Success(Some(persisted)) =>
+        val firstAccountKeys = (0 until persisted.accountKeysCount).map(i => derivePrivateKey(accountMaster, i)).toVector
+        val firstChangeKeys = (0 until persisted.changeKeysCount).map(i => derivePrivateKey(changeMaster, i)).toVector
+
+        Data(blockchain1,
+          firstAccountKeys,
+          firstChangeKeys,
+          status = persisted.status,
+          transactions = persisted.transactions,
+          heights = persisted.heights,
+          history = persisted.history,
+          proofs = persisted.proofs,
+          locks = persisted.locks,
+          pendingHistoryRequests = Set(),
+          pendingHeadersRequests = Set(),
+          pendingTransactionRequests = Set(),
+          pendingTransactions = persisted.pendingTransactions,
+          lastReadyMessage = None)
+      case _ =>
+        log.info("starting with a default wallet")
         val firstAccountKeys = (0 until params.swipeRange).map(i => derivePrivateKey(accountMaster, i)).toVector
         val firstChangeKeys = (0 until params.swipeRange).map(i => derivePrivateKey(changeMaster, i)).toVector
-    val transactions = walletDb.getTransactions().map(_._1)
+        Data(params, blockchain1, firstAccountKeys, firstChangeKeys)
-    log.info(s"loading ${transactions.size} transactions from db")
+    }
-    val txs = transactions.map(tx => tx.txid -> tx).toMap
-    val data = Data(params, blockchain1, firstAccountKeys, firstChangeKeys).copy(transactions = txs)
     context.system.eventStream.publish(NewWalletReceiveAddress(data.currentReceiveAddress))
+    log.info(s"restored wallet balance=${data.balance}")
     data
   })
 
@@ -124,17 +155,19 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
         log.info("performing full sync")
         // now ask for the first header after our latest checkpoint
         client ! ElectrumClient.GetHeaders(data.blockchain.checkpoints.size * RETARGETING_PERIOD, RETARGETING_PERIOD)
-        // make sure there is not last ready message
+        goto(SYNCING) using data
-        goto(SYNCING) using data.copy(lastReadyMessage = None)
       } else if (header == data.blockchain.tip.header) {
         // nothing to sync
         data.accountKeys.foreach(key => client ! ElectrumClient.ScriptHashSubscription(computeScriptHashFromPublicKey(key.publicKey), self))
         data.changeKeys.foreach(key => client ! ElectrumClient.ScriptHashSubscription(computeScriptHashFromPublicKey(key.publicKey), self))
-        goto(RUNNING) using notifyReady(data.copy(lastReadyMessage = None))
+        advertiseTransactions(data)
+        // tell everyone we're ready
+        goto(RUNNING) using persistAndNotify(data)
       } else {
         client ! ElectrumClient.GetHeaders(data.blockchain.tip.height + 1, RETARGETING_PERIOD)
-        log.info(s"syncing headers from ${data.blockchain.height} to ${height}")
+        log.info(s"syncing headers from ${data.blockchain.height} to ${height}, ready=${data.isReady(params.swipeRange)}")
-        goto(SYNCING) using data.copy(lastReadyMessage = None)
+        // tell everyone we're ready while we catch up
+        goto(SYNCING) using persistAndNotify(data)
       }
   }
 
@@ -145,7 +178,8 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
         log.info(s"headers sync complete, tip=${data.blockchain.tip}")
         data.accountKeys.foreach(key => client ! ElectrumClient.ScriptHashSubscription(computeScriptHashFromPublicKey(key.publicKey), self))
         data.changeKeys.foreach(key => client ! ElectrumClient.ScriptHashSubscription(computeScriptHashFromPublicKey(key.publicKey), self))
-        goto(RUNNING) using notifyReady(data)
+        advertiseTransactions(data)
+        goto(RUNNING) using persistAndNotify(data)
       } else {
         Try(Blockchain.addHeaders(data.blockchain, start, headers)) match {
           case Success(blockchain1) =>
@@ -185,11 +219,11 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
             data.heights.collect {
               case (txid, txheight) if txheight > 0 =>
                 val confirmations = computeDepth(height, txheight)
-                context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations))
+                context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations, data.computeTimestamp(txid, params.walletDb)))
             }
             val (blockchain2, saveme) = Blockchain.optimize(blockchain1)
             saveme.grouped(RETARGETING_PERIOD).foreach(chunk => params.walletDb.addHeaders(chunk.head.height, chunk.map(_.header)))
-            stay using notifyReady(data.copy(blockchain = blockchain2))
+            stay using persistAndNotify(data.copy(blockchain = blockchain2))
           case Failure(error) =>
             log.error(error, s"electrum server sent bad header, disconnecting")
             sender ! PoisonPill
@@ -198,7 +232,7 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
       }
 
     case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) if data.status.get(scriptHash) == Some(status) =>
-      stay using notifyReady(data)// we already have it
+      stay using persistAndNotify(data) // we already have it
 
     case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) if !data.accountKeyMap.contains(scriptHash) && !data.changeKeyMap.contains(scriptHash) =>
       log.warning(s"received status=$status for scriptHash=$scriptHash which does not match any of our keys")
@@ -206,7 +240,7 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
 
     case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) if status == "" =>
       val data1 = data.copy(status = data.status + (scriptHash -> status)) // empty status, nothing to do
-      stay using notifyReady(data1)
+      stay using persistAndNotify(data1)
 
     case Event(ElectrumClient.ScriptHashSubscriptionResponse(scriptHash, status), data) =>
       val key = data.accountKeyMap.getOrElse(scriptHash, data.changeKeyMap(scriptHash))
@@ -234,7 +268,7 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
         status = data.status + (scriptHash -> status),
         pendingHistoryRequests = data.pendingHistoryRequests + scriptHash)
 
-      stay using notifyReady(data1)
+      stay using persistAndNotify(data1)
 
     case Event(ElectrumClient.GetScriptHashHistoryResponse(scriptHash, items), data) =>
       log.debug(s"scriptHash=$scriptHash has history=$items")
@@ -245,15 +279,35 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
       shadow_items.foreach(item => log.warning(s"keeping shadow item for txid=${item.tx_hash}"))
       val items0 = items ++ shadow_items
 
+      val pendingHeadersRequests1 = collection.mutable.HashSet.empty[GetHeaders]
+      pendingHeadersRequests1 ++= data.pendingHeadersRequests
+
+      /**
+        * If we don't already have a header at this height, or a pending request to download the header chunk it's in,
+        * download this header chunk.
+        * We don't have this header because it's most likely older than our current checkpoint, downloading the whole header
+        * chunk (2016 headers) is quick and they're easy to verify.
+        */
+      def downloadHeadersIfMissing(height: Int): Unit = {
+        if (data.blockchain.getHeader(height).orElse(params.walletDb.getHeader(height)).isEmpty) {
+          // we don't have this header, probably because it is older than our checkpoints
+          // request the entire chunk, we will be able to check it efficiently and then store it
+          val start = (height / RETARGETING_PERIOD) * RETARGETING_PERIOD
+          val request = GetHeaders(start, RETARGETING_PERIOD)
+          // there may be already a pending request for this chunk of headers
+          if (!pendingHeadersRequests1.contains(request)) {
+            client ! request
+            pendingHeadersRequests1.add(request)
+          }
+        }
+      }
+
       val (heights1, pendingTransactionRequests1) = items0.foldLeft((data.heights, data.pendingTransactionRequests)) {
         case ((heights, hashes), item) if !data.transactions.contains(item.tx_hash) && !data.pendingTransactionRequests.contains(item.tx_hash) =>
           // we retrieve the tx if we don't have it and haven't yet requested it
           client ! GetTransaction(item.tx_hash)
           if (item.height > 0) { // don't ask for merkle proof for unconfirmed transactions
-            if (data.blockchain.getHeader(item.height).orElse(params.walletDb.getHeader(item.height)).isEmpty) {
+            downloadHeadersIfMissing(item.height)
-              val start = (item.height / RETARGETING_PERIOD) * RETARGETING_PERIOD
-              client ! GetHeaders(start, RETARGETING_PERIOD)
-            }
             client ! GetMerkle(item.tx_hash, item.height)
           }
           (heights + (item.tx_hash -> item.height), hashes + item.tx_hash)
@@ -271,54 +325,84 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
             // height=0 => unconfirmed, height=-1 => unconfirmed and one input is unconfirmed
             case (None, height) if height > 0 =>
               // first time we get a height for this tx: either it was just confirmed, or we restarted the wallet
-              context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations))
+              context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations, data.computeTimestamp(txid, params.walletDb)))
+              downloadHeadersIfMissing(height.toInt)
+              client ! GetMerkle(txid, height.toInt)
             case (Some(previousHeight), height) if previousHeight != height =>
               // there was a reorg
-              context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations))
+              context.system.eventStream.publish(TransactionConfidenceChanged(txid, confirmations, data.computeTimestamp(txid, params.walletDb)))
+              downloadHeadersIfMissing(height.toInt)
+              client ! GetMerkle(txid, height.toInt)
+            case (Some(previousHeight), height) if previousHeight == height && data.proofs.get(txid).isEmpty =>
+              downloadHeadersIfMissing(height.toInt)
+              client ! GetMerkle(txid, height.toInt)
             case (Some(previousHeight), height) if previousHeight == height =>
             // no reorg, nothing to do
           }
       }
-      val data1 = data.copy(heights = heights1, history = data.history + (scriptHash -> items0), pendingHistoryRequests = data.pendingHistoryRequests - scriptHash, pendingTransactionRequests = pendingTransactionRequests1)
+      val data1 = data.copy(
-      stay using notifyReady(data1)
+        heights = heights1,
+        history = data.history + (scriptHash -> items0),
+        pendingHistoryRequests = data.pendingHistoryRequests - scriptHash,
+        pendingTransactionRequests = pendingTransactionRequests1,
+        pendingHeadersRequests = pendingHeadersRequests1.toSet)
+      stay using persistAndNotify(data1)
+
+    case Event(ElectrumClient.GetHeadersResponse(start, headers, _), data) =>
+      Try(Blockchain.addHeadersChunk(data.blockchain, start, headers)) match {
+        case Success(blockchain1) =>
+          params.walletDb.addHeaders(start, headers)
+          stay() using data.copy(blockchain = blockchain1)
+        case Failure(error) =>
+          log.error("electrum server sent bad headers, disconnecting", error)
+          sender ! PoisonPill
+          goto(DISCONNECTED) using data
+      }
 
     case Event(GetTransactionResponse(tx), data) =>
       log.debug(s"received transaction ${tx.txid}")
       data.computeTransactionDelta(tx) match {
         case Some((received, sent, fee_opt)) =>
           log.info(s"successfully connected txid=${tx.txid}")
-          context.system.eventStream.publish(TransactionReceived(tx, data.computeTransactionDepth(tx.txid), received, sent, fee_opt))
+          context.system.eventStream.publish(TransactionReceived(tx, data.computeTransactionDepth(tx.txid), received, sent, fee_opt, data.computeTimestamp(tx.txid, params.walletDb)))
           // when we have successfully processed a new tx, we retry all pending txes to see if they can be added now
           data.pendingTransactions.foreach(self ! GetTransactionResponse(_))
           val data1 = data.copy(transactions = data.transactions + (tx.txid -> tx), pendingTransactionRequests = data.pendingTransactionRequests - tx.txid, pendingTransactions = Nil)
-          stay using notifyReady(data1)
+          stay using persistAndNotify(data1)
         case None =>
           // missing parents
           log.info(s"couldn't connect txid=${tx.txid}")
           val data1 = data.copy(pendingTransactions = data.pendingTransactions :+ tx)
-          stay using notifyReady(data1)
+          stay using persistAndNotify(data1)
       }
 
     case Event(response@GetMerkleResponse(txid, _, height, _), data) =>
       data.blockchain.getHeader(height).orElse(params.walletDb.getHeader(height)) match {
         case Some(header) if header.hashMerkleRoot == response.root =>
           log.info(s"transaction $txid has been verified")
-          data.transactions.get(txid).orElse(data.pendingTransactions.find(_.txid == txid)) match {
+          val data1 = if (data.transactions.get(txid).isEmpty && !data.pendingTransactionRequests.contains(txid) && !data.pendingTransactions.exists(_.txid == txid)) {
-            case Some(tx) =>
+            log.warning(s"we received a Merkle proof for transaction $txid that we don't have")
-              log.info(s"saving ${tx.txid} to our db")
+            data
-              walletDb.addTransaction(tx, response)
+          } else {
-            case None => log.warning(s"we received a Merkle proof for transaction $txid that we don't have")
+            data.copy(proofs = data.proofs + (txid -> response))
           }
-          stay()
+          stay using data1
-        case Some(header) =>
+        case Some(_) =>
           log.error(s"server sent an invalid proof for $txid, disconnecting")
           sender ! PoisonPill
           stay() using data.copy(transactions = data.transactions - txid)
         case None =>
           // this is probably because the tx is old and within our checkpoints => request the whole header chunk
           val start = (height / RETARGETING_PERIOD) * RETARGETING_PERIOD
-          client ! GetHeaders(start, RETARGETING_PERIOD)
+          val request = GetHeaders(start, RETARGETING_PERIOD)
-          stay()
+          val pendingHeadersRequest1 = if (data.pendingHeadersRequests.contains(request)) {
+            data.pendingHeadersRequests
+          } else {
+            client ! request
+            self ! response
+            data.pendingHeadersRequests + request
+          }
+          stay() using data.copy(pendingHeadersRequests = pendingHeadersRequest1)
       }
 
     case Event(CompleteTransaction(tx, feeRatePerKw), data) =>
@@ -339,12 +423,12 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
       // we know all the parents
       val (received, sent, Some(fee)) = data.computeTransactionDelta(tx).get
       // we notify here because the tx won't be downloaded again (it has been added to the state at commit)
-      context.system.eventStream.publish(TransactionReceived(tx, data1.computeTransactionDepth(tx.txid), received, sent, Some(fee)))
+      context.system.eventStream.publish(TransactionReceived(tx, data1.computeTransactionDepth(tx.txid), received, sent, Some(fee), None))
-      stay using notifyReady(data1) replying CommitTransactionResponse(tx) // goto instead of stay because we want to fire transitions
+      stay using persistAndNotify(data1) replying CommitTransactionResponse(tx) // goto instead of stay because we want to fire transitions
 
     case Event(CancelTransaction(tx), data) =>
       log.info(s"cancelling txid=${tx.txid}")
-      stay using notifyReady(data.cancelTransaction(tx)) replying CancelTransactionResponse(tx)
+      stay using persistAndNotify(data.cancelTransaction(tx)) replying CancelTransactionResponse(tx)
 
     case Event(bc@ElectrumClient.BroadcastTransaction(tx), _) =>
       log.info(s"broadcasting txid=${tx.txid}")
@@ -359,10 +443,8 @@ class ElectrumWallet(seed: BinaryData, client: ActorRef, params: ElectrumWallet.
       goto(DISCONNECTED) using data.copy(
         pendingHistoryRequests = Set(),
         pendingTransactionRequests = Set(),
-        pendingTransactions = Seq(),
+        pendingHeadersRequests = Set(),
-        status = Map(),
+        lastReadyMessage = None
-        heights = Map(),
-        history = Map()
       )
 
     case Event(GetCurrentReceiveAddress, data) => stay replying GetCurrentReceiveAddressResponse(data.currentReceiveAddress)
@@ -443,8 +525,8 @@ object ElectrumWallet {
     * @param sent
     * @param feeOpt is set only when we know it (i.e. for outgoing transactions)
     */
-  case class TransactionReceived(tx: Transaction, depth: Long, received: Satoshi, sent: Satoshi, feeOpt: Option[Satoshi]) extends WalletEvent
+  case class TransactionReceived(tx: Transaction, depth: Long, received: Satoshi, sent: Satoshi, feeOpt: Option[Satoshi], timestamp: Option[Long]) extends WalletEvent
-  case class TransactionConfidenceChanged(txid: BinaryData, depth: Long) extends WalletEvent
+  case class TransactionConfidenceChanged(txid: BinaryData, depth: Long, timestamp: Option[Long]) extends WalletEvent
   case class NewWalletReceiveAddress(address: String) extends WalletEvent
   case class WalletReady(confirmedBalance: Satoshi, unconfirmedBalance: Satoshi, height: Long, timestamp: Long) extends WalletEvent
   // @formatter:on
@@ -497,6 +579,7 @@ object ElectrumWallet {
 
   /**
     * Compute the wallet's xpub
+    *
     * @param master    master key
     * @param chainHash chain hash
     * @return a (xpub, path) tuple where xpub is the encoded account public key, and path is the derivation path for the account key
@@ -583,11 +666,13 @@ object ElectrumWallet {
                   status: Map[BinaryData, String],
                   transactions: Map[BinaryData, Transaction],
                   heights: Map[BinaryData, Long],
-                  history: Map[BinaryData, Seq[ElectrumClient.TransactionHistoryItem]],
+                  history: Map[BinaryData, List[ElectrumClient.TransactionHistoryItem]],
+                  proofs: Map[BinaryData, GetMerkleResponse],
                   locks: Set[Transaction],
                   pendingHistoryRequests: Set[BinaryData],
                   pendingTransactionRequests: Set[BinaryData],
-                  pendingTransactions: Seq[Transaction],
+                  pendingHeadersRequests: Set[GetHeaders],
+                  pendingTransactions: List[Transaction],
                   lastReadyMessage: Option[WalletReady]) extends Logging {
     val chainHash = blockchain.chainHash
 
@@ -663,6 +748,19 @@ object ElectrumWallet {
 
     def computeTransactionDepth(txid: BinaryData): Long = heights.get(txid).map(height => if (height > 0) computeDepth(blockchain.tip.height, height) else 0).getOrElse(0)
 
+    /**
+      *
+      * @param txid     transaction id
+      * @param headerDb header db
+      * @return the timestamp of the block this tx was included in
+      */
+    def computeTimestamp(txid: BinaryData, headerDb: HeaderDb): Option[Long] = {
+      for {
+        height <- heights.get(txid).map(_.toInt)
+        header <- blockchain.getHeader(height).orElse(headerDb.getHeader(height))
+      } yield header.time
+    }
+
     /**
       *
       * @param scriptHash script hash
@@ -896,9 +994,9 @@ object ElectrumWallet {
         .foldLeft(this.history) {
           case (history, scriptHash) =>
             val entry = history.get(scriptHash) match {
-              case None => Seq(TransactionHistoryItem(0, tx.txid))
+              case None => List(TransactionHistoryItem(0, tx.txid))
               case Some(items) if items.map(_.tx_hash).contains(tx.txid) => items
-              case Some(items) => items :+ TransactionHistoryItem(0, tx.txid)
+              case Some(items) => TransactionHistoryItem(0, tx.txid) :: items
             }
             history + (scriptHash -> entry)
         }
@@ -908,6 +1006,7 @@ object ElectrumWallet {
     /**
       * spend all our balance, including unconfirmed utxos and locked utxos (i.e utxos
       * that are used in funding transactions that have not been published yet
+      *
       * @param publicKeyScript script to send all our funds to
       * @param feeRatePerKw    fee rate in satoshi per kiloweight
       * @return a (tx, fee) tuple, tx is a signed transaction that spends all our balance and
@@ -930,9 +1029,23 @@ object ElectrumWallet {
 
   object Data {
     def apply(params: ElectrumWallet.WalletParameters, blockchain: Blockchain, accountKeys: Vector[ExtendedPrivateKey], changeKeys: Vector[ExtendedPrivateKey]): Data
-    = Data(blockchain, accountKeys, changeKeys, Map(), Map(), Map(), Map(), Set(), Set(), Set(), Seq(), None)
+    = Data(blockchain, accountKeys, changeKeys, Map(), Map(), Map(), Map(), Map(), Set(), Set(), Set(), Set(), List(), None)
   }
 
   case class InfiniteLoopException(data: Data, tx: Transaction) extends Exception
 
+  case class PersistentData(accountKeysCount: Int,
+                            changeKeysCount: Int,
+                            status: Map[BinaryData, String],
+                            transactions: Map[BinaryData, Transaction],
+                            heights: Map[BinaryData, Long],
+                            history: Map[BinaryData, List[ElectrumClient.TransactionHistoryItem]],
+                            proofs: Map[BinaryData, GetMerkleResponse],
+                            pendingTransactions: List[Transaction],
+                            locks: Set[Transaction])
+
+  object PersistentData {
+    def apply(data: Data) = new PersistentData(data.accountKeys.length, data.changeKeys.length, data.status, data.transactions, data.heights, data.history, data.proofs, data.pendingTransactions, data.locks)
+  }
+
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWatcher.scala

@@ -44,7 +44,7 @@ class ElectrumWatcher(client: ActorRef) extends Actor with Stash with ActorLoggi
           txIn = Seq.empty[TxIn],
           txOut = List.fill(outputIndex + 1)(TxOut(Satoshi(0), pubkeyScript)), // quick and dirty way to be sure that the outputIndex'th output is of the expected format
           lockTime = 0)
-      sender ! ValidateResult(c, Some(fakeFundingTx), true, None)
+      sender ! ValidateResult(c, Right((fakeFundingTx, UtxoStatus.Unspent)))
 
     case _ => log.warning(s"unhandled message $message")
   }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/db/WalletDb.scala

@@ -1,7 +1,24 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.blockchain.electrum.db
 
 import fr.acinq.bitcoin.{BinaryData, BlockHeader, Transaction}
 import fr.acinq.eclair.blockchain.electrum.ElectrumClient.GetMerkleResponse
+import fr.acinq.eclair.blockchain.electrum.ElectrumWallet.PersistentData
 
 trait HeaderDb {
   def addHeader(height: Int, header: BlockHeader): Unit
@@ -18,12 +35,8 @@ trait HeaderDb {
   def getTip: Option[(Int, BlockHeader)]
 }
 
-trait TransactionDb {
+trait WalletDb extends HeaderDb {
-  def addTransaction(tx: Transaction, proof: GetMerkleResponse): Unit
+  def persist(data: PersistentData): Unit
 
-  def getTransaction(txid: BinaryData): Option[(Transaction, GetMerkleResponse)]
+  def readPersistentData(): Option[PersistentData]
-
-  def getTransactions(): Seq[(Transaction, GetMerkleResponse)]
 }
-
-trait WalletDb extends HeaderDb with TransactionDb

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/electrum/db/sqlite/SqliteWalletDb.scala

@@ -1,10 +1,27 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.blockchain.electrum.db.sqlite
 
 import java.sql.Connection
 
 import fr.acinq.bitcoin.{BinaryData, BlockHeader, Transaction}
-import fr.acinq.eclair.blockchain.electrum.ElectrumClient
+import fr.acinq.eclair.blockchain.electrum.{ElectrumClient, ElectrumWallet}
-import fr.acinq.eclair.blockchain.electrum.ElectrumClient.GetMerkleResponse
+import fr.acinq.eclair.blockchain.electrum.ElectrumClient.{GetMerkleResponse, TransactionHistoryItem}
+import fr.acinq.eclair.blockchain.electrum.ElectrumWallet.PersistentData
 import fr.acinq.eclair.blockchain.electrum.db.WalletDb
 import fr.acinq.eclair.db.sqlite.SqliteUtils
 
@@ -16,7 +33,7 @@ class SqliteWalletDb(sqlite: Connection) extends WalletDb {
 
   using(sqlite.createStatement()) { statement =>
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS headers (height INTEGER NOT NULL PRIMARY KEY, block_hash BLOB NOT NULL, header BLOB NOT NULL)")
-    statement.executeUpdate("CREATE TABLE IF NOT EXISTS transactions (tx_hash BLOB PRIMARY KEY, tx BLOB NOT NULL, proof BLOB NOT NULL)")
+    statement.executeUpdate("CREATE TABLE IF NOT EXISTS wallet (data BLOB)")
   }
 
   override def addHeader(height: Int, header: BlockHeader): Unit = {
@@ -91,41 +108,35 @@ class SqliteWalletDb(sqlite: Connection) extends WalletDb {
     }
   }
 
-  override def addTransaction(tx: Transaction, proof: ElectrumClient.GetMerkleResponse): Unit = {
+  override def persist(data: ElectrumWallet.PersistentData): Unit = {
-    using(sqlite.prepareStatement("INSERT OR IGNORE INTO transactions VALUES (?, ?, ?)")) { statement =>
+    val bin = SqliteWalletDb.serialize(data)
-      statement.setBytes(1, tx.hash)
+    using(sqlite.prepareStatement("UPDATE wallet SET data=(?)")) { update =>
-      statement.setBytes(2, Transaction.write(tx))
+      update.setBytes(1, bin)
-      statement.setBytes(3, SqliteWalletDb.serialize(proof))
+      if (update.executeUpdate() == 0) {
+        using(sqlite.prepareStatement("INSERT INTO wallet VALUES (?)")) { statement =>
+          statement.setBytes(1, bin)
           statement.executeUpdate()
         }
       }
+    }
+  }
 
-  override def getTransaction(tx_hash: BinaryData): Option[(Transaction, ElectrumClient.GetMerkleResponse)] = {
+  override def readPersistentData(): Option[ElectrumWallet.PersistentData] = {
-    using(sqlite.prepareStatement("SELECT tx, proof FROM transactions WHERE tx_hash = ?")) { statement =>
+    using(sqlite.prepareStatement("SELECT data FROM wallet")) { statement =>
-      statement.setBytes(1, tx_hash)
       val rs = statement.executeQuery()
       if (rs.next()) {
-        Some((Transaction.read(rs.getBytes("tx")), SqliteWalletDb.deserialize((rs.getBytes("proof")))))
+        Option(rs.getBytes(1)).map(bin => SqliteWalletDb.deserializePersistentData(BinaryData(bin)))
       } else {
         None
       }
     }
   }
-
-  override def getTransactions(): Seq[(Transaction, ElectrumClient.GetMerkleResponse)] = {
-    using(sqlite.prepareStatement("SELECT tx, proof FROM transactions")) { statement =>
-      val rs = statement.executeQuery()
-      var q: Queue[(Transaction, ElectrumClient.GetMerkleResponse)] = Queue()
-      while (rs.next()) {
-        q = q :+ (Transaction.read(rs.getBytes("tx")), SqliteWalletDb.deserialize(rs.getBytes("proof")))
-      }
-      q
-    }
-  }
 }
 
 object SqliteWalletDb {
-  import fr.acinq.eclair.wire.LightningMessageCodecs.binarydata
+
+  import fr.acinq.eclair.wire.LightningMessageCodecs._
+  import fr.acinq.eclair.wire.ChannelCodecs._
   import scodec.Codec
   import scodec.bits.BitVector
   import scodec.codecs._
@@ -136,7 +147,65 @@ object SqliteWalletDb {
       ("block_height" | uint24) ::
       ("pos" | uint24)).as[GetMerkleResponse]
 
-  def serialize(proof: GetMerkleResponse) : BinaryData = proofCodec.encode(proof).require.toByteArray
+  def serializeMerkleProof(proof: GetMerkleResponse): BinaryData = proofCodec.encode(proof).require.toByteArray
 
-  def deserialize(bin: BinaryData) : GetMerkleResponse = proofCodec.decode(BitVector(bin.toArray)).require.value
+  def deserializeMerkleProof(bin: BinaryData): GetMerkleResponse = proofCodec.decode(BitVector(bin.toArray)).require.value
+
+  import fr.acinq.eclair.wire.LightningMessageCodecs._
+
+  val statusListCodec: Codec[List[(BinaryData, String)]] = listOfN(uint16, binarydata(32) ~ cstring)
+
+  val statusCodec: Codec[Map[BinaryData, String]] = Codec[Map[BinaryData, String]](
+    (map: Map[BinaryData, String]) => statusListCodec.encode(map.toList),
+    (wire: BitVector) => statusListCodec.decode(wire).map(_.map(_.toMap))
+  )
+
+  val heightsListCodec: Codec[List[(BinaryData, Long)]] = listOfN(uint16, binarydata(32) ~ uint32)
+
+  val heightsCodec: Codec[Map[BinaryData, Long]] = Codec[Map[BinaryData, Long]](
+    (map: Map[BinaryData, Long]) => heightsListCodec.encode(map.toList),
+    (wire: BitVector) => heightsListCodec.decode(wire).map(_.map(_.toMap))
+  )
+
+  val transactionListCodec: Codec[List[(BinaryData, Transaction)]] = listOfN(uint16, binarydata(32) ~ txCodec)
+
+  val transactionsCodec: Codec[Map[BinaryData, Transaction]] = Codec[Map[BinaryData, Transaction]](
+    (map: Map[BinaryData, Transaction]) => transactionListCodec.encode(map.toList),
+    (wire: BitVector) => transactionListCodec.decode(wire).map(_.map(_.toMap))
+  )
+
+  val transactionHistoryItemCodec: Codec[ElectrumClient.TransactionHistoryItem] = (
+    ("height" | int32) :: ("tx_hash" | binarydata(size = 32))).as[ElectrumClient.TransactionHistoryItem]
+
+  val seqOfTransactionHistoryItemCodec: Codec[List[TransactionHistoryItem]] = listOfN[TransactionHistoryItem](uint16, transactionHistoryItemCodec)
+
+  val historyListCodec: Codec[List[(BinaryData, List[ElectrumClient.TransactionHistoryItem])]] =
+    listOfN[(BinaryData, List[ElectrumClient.TransactionHistoryItem])](uint16, binarydata(32) ~ seqOfTransactionHistoryItemCodec)
+
+  val historyCodec: Codec[Map[BinaryData, List[ElectrumClient.TransactionHistoryItem]]] = Codec[Map[BinaryData, List[ElectrumClient.TransactionHistoryItem]]](
+    (map: Map[BinaryData, List[ElectrumClient.TransactionHistoryItem]]) => historyListCodec.encode(map.toList),
+    (wire: BitVector) => historyListCodec.decode(wire).map(_.map(_.toMap))
+  )
+
+  val proofsListCodec: Codec[List[(BinaryData, GetMerkleResponse)]] = listOfN(uint16, binarydata(32) ~ proofCodec)
+
+  val proofsCodec: Codec[Map[BinaryData, GetMerkleResponse]] = Codec[Map[BinaryData, GetMerkleResponse]](
+    (map: Map[BinaryData, GetMerkleResponse]) => proofsListCodec.encode(map.toList),
+    (wire: BitVector) => proofsListCodec.decode(wire).map(_.map(_.toMap))
+  )
+
+  val persistentDataCodec: Codec[PersistentData] = (
+    ("accountKeysCount" | int32) ::
+      ("accountKeysCount" | int32) ::
+      ("status" | statusCodec) ::
+      ("transactions" | transactionsCodec) ::
+      ("heights" | heightsCodec) ::
+      ("history" | historyCodec) ::
+      ("proofs" | proofsCodec) ::
+      ("pendingTransactions" | listOfN(uint16, txCodec)) ::
+      ("locks" | setCodec(txCodec))).as[PersistentData]
+
+  def serialize(data: PersistentData): BinaryData = persistentDataCodec.encode(data).require.toByteArray
+
+  def deserializePersistentData(bin: BinaryData): PersistentData = persistentDataCodec.decode(BitVector(bin.toArray)).require.value
 }

eclair-core/src/main/scala/fr/acinq/eclair/blockchain/fee/SmoothFeeProvider.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.blockchain.fee
 
 import scala.concurrent.{ExecutionContext, Future}

eclair-core/src/main/scala/fr/acinq/eclair/channel/Channel.scala

@@ -633,7 +633,10 @@ class Channel(val nodeParams: NodeParams, wallet: EclairWallet, remoteNodeId: Pu
                 self ! TickRefreshChannelUpdate
               }
               context.system.eventStream.publish(ChannelSignatureSent(self, commitments1))
-              context.system.eventStream.publish(AvailableBalanceChanged(self, d.channelId, d.shortChannelId, nextRemoteCommit.spec.toRemoteMsat)) // note that remoteCommit.toRemote == toLocal
+              if (nextRemoteCommit.spec.toRemoteMsat != d.commitments.remoteCommit.spec.toRemoteMsat) {
+                // we send this event only when our balance changes (note that remoteCommit.toRemote == toLocal)
+                context.system.eventStream.publish(AvailableBalanceChanged(self, d.channelId, d.shortChannelId, nextRemoteCommit.spec.toRemoteMsat, commitments1))
+              }
               // we expect a quick response from our peer
               setTimer(RevocationTimeout.toString, RevocationTimeout(commitments1.remoteCommit.index, peer = context.parent), timeout = nodeParams.revocationTimeout, repeat = false)
               handleCommandSuccess(sender, store(d.copy(commitments = commitments1))) sending commit
@@ -1249,6 +1252,7 @@ class Channel(val nodeParams: NodeParams, wallet: EclairWallet, remoteNodeId: Pu
       closeType_opt match {
         case Some(closeType) =>
           log.info(s"channel closed (type=$closeType)")
+          context.system.eventStream.publish(ChannelClosed(self, d.channelId, closeType, d.commitments))
           goto(CLOSED) using store(d1)
         case None =>
           stay using store(d1)

eclair-core/src/main/scala/fr/acinq/eclair/channel/ChannelEvents.scala

@@ -52,8 +52,10 @@ case class ChannelFailed(channel: ActorRef, channelId: BinaryData, remoteNodeId:
 case class NetworkFeePaid(channel: ActorRef, remoteNodeId: PublicKey, channelId: BinaryData, tx: Transaction, fee: Satoshi, txType: String) extends ChannelEvent
 
 // NB: this event is only sent when the channel is available
-case class AvailableBalanceChanged(channel: ActorRef, channelId: BinaryData, shortChannelId: ShortChannelId, localBalanceMsat: Long) extends ChannelEvent
+case class AvailableBalanceChanged(channel: ActorRef, channelId: BinaryData, shortChannelId: ShortChannelId, localBalanceMsat: Long, commitments: Commitments) extends ChannelEvent
 
 case class ChannelPersisted(channel: ActorRef, remoteNodeId: PublicKey, channelId: BinaryData, data: Data) extends ChannelEvent
 
 case class LocalCommitConfirmed(channel: ActorRef, remoteNodeId: PublicKey, channelId: BinaryData, refundAtBlock: Long) extends ChannelEvent
+
+case class ChannelClosed(channel: ActorRef, channelId: BinaryData, closeType: String, commitments: Commitments)

eclair-core/src/main/scala/fr/acinq/eclair/channel/Commitments.scala

@@ -71,6 +71,12 @@ case class Commitments(localParams: LocalParams, remoteParams: RemoteParams,
   def addRemoteProposal(proposal: UpdateMessage): Commitments = Commitments.addRemoteProposal(this, proposal)
 
   def announceChannel: Boolean = (channelFlags & 0x01) != 0
+
+  def availableBalanceForSendMsat: Long = {
+    val reduced = CommitmentSpec.reduce(remoteCommit.spec, remoteChanges.acked, localChanges.proposed)
+    val fees = if (localParams.isFunder) Transactions.commitTxFee(Satoshi(remoteParams.dustLimitSatoshis), reduced).amount else 0
+    reduced.toRemoteMsat / 1000 - remoteParams.channelReserveSatoshis - fees
+  }
 }
 
 object Commitments {

eclair-core/src/main/scala/fr/acinq/eclair/db/AuditDb.scala

@@ -18,11 +18,15 @@ package fr.acinq.eclair.db
 
 import fr.acinq.bitcoin.BinaryData
 import fr.acinq.bitcoin.Crypto.PublicKey
-import fr.acinq.eclair.channel.NetworkFeePaid
+import fr.acinq.eclair.channel._
 import fr.acinq.eclair.payment.{PaymentReceived, PaymentRelayed, PaymentSent}
 
 trait AuditDb {
 
+  def add(availableBalanceChanged: AvailableBalanceChanged)
+
+  def add(channelLifecycle: ChannelLifecycleEvent)
+
   def add(paymentSent: PaymentSent)
 
   def add(paymentReceived: PaymentReceived)
@@ -45,6 +49,8 @@ trait AuditDb {
 
 }
 
+case class ChannelLifecycleEvent(channelId: BinaryData, remoteNodeId: PublicKey, capacitySat: Long, isFunder: Boolean, isPrivate: Boolean, event: String)
+
 case class NetworkFee(remoteNodeId: PublicKey, channelId: BinaryData, txId: BinaryData, feeSat: Long, txType: String, timestamp: Long)
 
 case class Stats(channelId: BinaryData, avgPaymentAmountSatoshi: Long, paymentCount: Int, relayFeeSatoshi: Long, networkFeeSatoshi: Long)

eclair-core/src/main/scala/fr/acinq/eclair/db/PeersDb.scala

@@ -16,17 +16,16 @@
 
 package fr.acinq.eclair.db
 
-import java.net.InetSocketAddress
-
 import fr.acinq.bitcoin.Crypto.PublicKey
+import fr.acinq.eclair.wire.NodeAddress
 
 trait PeersDb {
 
-  def addOrUpdatePeer(nodeId: PublicKey, address: InetSocketAddress)
+  def addOrUpdatePeer(nodeId: PublicKey, address: NodeAddress)
 
   def removePeer(nodeId: PublicKey)
 
-  def listPeers(): Map[PublicKey, InetSocketAddress]
+  def listPeers(): Map[PublicKey, NodeAddress]
 
   def close(): Unit
 

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqliteAuditDb.scala

@@ -19,9 +19,9 @@ package fr.acinq.eclair.db.sqlite
 import java.sql.Connection
 
 import fr.acinq.bitcoin.Crypto.PublicKey
-import fr.acinq.bitcoin.{BinaryData, MilliSatoshi, Satoshi}
+import fr.acinq.bitcoin.{BinaryData, MilliSatoshi}
-import fr.acinq.eclair.channel.NetworkFeePaid
+import fr.acinq.eclair.channel.{AvailableBalanceChanged, ChannelClosed, ChannelCreated, NetworkFeePaid}
-import fr.acinq.eclair.db.{AuditDb, NetworkFee, Stats}
+import fr.acinq.eclair.db.{AuditDb, ChannelLifecycleEvent, NetworkFee, Stats}
 import fr.acinq.eclair.payment.{PaymentReceived, PaymentRelayed, PaymentSent}
 
 import scala.collection.immutable.Queue
@@ -36,15 +36,42 @@ class SqliteAuditDb(sqlite: Connection) extends AuditDb {
 
   using(sqlite.createStatement()) { statement =>
     require(getVersion(statement, DB_NAME, CURRENT_VERSION) == CURRENT_VERSION) // there is only one version currently deployed
+    statement.executeUpdate("CREATE TABLE IF NOT EXISTS balance_updated (channel_id BLOB NOT NULL, node_id BLOB NOT NULL, amount_msat INTEGER NOT NULL, capacity_sat INTEGER NOT NULL, reserve_sat INTEGER NOT NULL, timestamp INTEGER NOT NULL)")
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS sent (amount_msat INTEGER NOT NULL, fees_msat INTEGER NOT NULL, payment_hash BLOB NOT NULL, payment_preimage BLOB NOT NULL, to_channel_id BLOB NOT NULL, timestamp INTEGER NOT NULL)")
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS received (amount_msat INTEGER NOT NULL, payment_hash BLOB NOT NULL, from_channel_id BLOB NOT NULL, timestamp INTEGER NOT NULL)")
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS relayed (amount_in_msat INTEGER NOT NULL, amount_out_msat INTEGER NOT NULL, payment_hash BLOB NOT NULL, from_channel_id BLOB NOT NULL, to_channel_id BLOB NOT NULL, timestamp INTEGER NOT NULL)")
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS network_fees (channel_id BLOB NOT NULL, node_id BLOB NOT NULL, tx_id BLOB NOT NULL, fee_sat INTEGER NOT NULL, tx_type TEXT NOT NULL, timestamp INTEGER NOT NULL)")
+    statement.executeUpdate("CREATE TABLE IF NOT EXISTS channel_events (channel_id BLOB NOT NULL, node_id BLOB NOT NULL, capacity_sat INTEGER NOT NULL, is_funder BOOLEAN NOT NULL, is_private BOOLEAN NOT NULL, event STRING NOT NULL, timestamp INTEGER NOT NULL)")
 
+    statement.executeUpdate("CREATE INDEX IF NOT EXISTS balance_updated_idx ON balance_updated(timestamp)")
     statement.executeUpdate("CREATE INDEX IF NOT EXISTS sent_timestamp_idx ON sent(timestamp)")
     statement.executeUpdate("CREATE INDEX IF NOT EXISTS received_timestamp_idx ON received(timestamp)")
     statement.executeUpdate("CREATE INDEX IF NOT EXISTS relayed_timestamp_idx ON relayed(timestamp)")
     statement.executeUpdate("CREATE INDEX IF NOT EXISTS network_fees_timestamp_idx ON network_fees(timestamp)")
+    statement.executeUpdate("CREATE INDEX IF NOT EXISTS channel_events_timestamp_idx ON channel_events(timestamp)")
+  }
+
+  override def add(e: AvailableBalanceChanged): Unit =
+    using(sqlite.prepareStatement("INSERT INTO balance_updated VALUES (?, ?, ?, ?, ?, ?)")) { statement =>
+      statement.setBytes(1, e.channelId)
+      statement.setBytes(2, e.commitments.remoteParams.nodeId.toBin)
+      statement.setLong(3, e.localBalanceMsat)
+      statement.setLong(4, e.commitments.commitInput.txOut.amount.toLong)
+      statement.setLong(5, e.commitments.remoteParams.channelReserveSatoshis) // remote decides what our reserve should be
+      statement.setLong(6, Platform.currentTime)
+      statement.executeUpdate()
+    }
+
+  override def add(e: ChannelLifecycleEvent): Unit =
+    using(sqlite.prepareStatement("INSERT INTO channel_events VALUES (?, ?, ?, ?, ?, ?, ?)")) { statement =>
+      statement.setBytes(1, e.channelId)
+      statement.setBytes(2, e.remoteNodeId.toBin)
+      statement.setLong(3, e.capacitySat)
+      statement.setBoolean(4, e.isFunder)
+      statement.setBoolean(5, e.isPrivate)
+      statement.setString(6, e.event)
+      statement.setLong(7, Platform.currentTime)
+      statement.executeUpdate()
     }
 
   override def add(e: PaymentSent): Unit =
@@ -203,4 +230,5 @@ class SqliteAuditDb(sqlite: Connection) extends AuditDb {
     }
 
   override def close(): Unit = sqlite.close()
+
 }

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePeersDb.scala

@@ -16,14 +16,13 @@
 
 package fr.acinq.eclair.db.sqlite
 
-import java.net.{Inet4Address, Inet6Address, InetSocketAddress}
 import java.sql.Connection
 
 import fr.acinq.bitcoin.Crypto
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.eclair.db.PeersDb
 import fr.acinq.eclair.db.sqlite.SqliteUtils.{getVersion, using}
-import fr.acinq.eclair.wire.{IPv4, IPv6, LightningMessageCodecs, NodeAddress}
+import fr.acinq.eclair.wire._
 import scodec.bits.BitVector
 
 class SqlitePeersDb(sqlite: Connection) extends PeersDb {
@@ -36,8 +35,7 @@ class SqlitePeersDb(sqlite: Connection) extends PeersDb {
     statement.executeUpdate("CREATE TABLE IF NOT EXISTS peers (node_id BLOB NOT NULL PRIMARY KEY, data BLOB NOT NULL)")
   }
 
-  override def addOrUpdatePeer(nodeId: Crypto.PublicKey, address: InetSocketAddress): Unit = {
+  override def addOrUpdatePeer(nodeId: Crypto.PublicKey, nodeaddress: NodeAddress): Unit = {
-    val nodeaddress = NodeAddress(address)
     val data = LightningMessageCodecs.nodeaddress.encode(nodeaddress).require.toByteArray
     using(sqlite.prepareStatement("UPDATE peers SET data=? WHERE node_id=?")) { update =>
       update.setBytes(1, data)
@@ -59,17 +57,13 @@ class SqlitePeersDb(sqlite: Connection) extends PeersDb {
     }
   }
 
-  override def listPeers(): Map[PublicKey, InetSocketAddress] = {
+  override def listPeers(): Map[PublicKey, NodeAddress] = {
     using(sqlite.createStatement()) { statement =>
       val rs = statement.executeQuery("SELECT node_id, data FROM peers")
-      var m: Map[PublicKey, InetSocketAddress] = Map()
+      var m: Map[PublicKey, NodeAddress] = Map()
       while (rs.next()) {
         val nodeid = PublicKey(rs.getBytes("node_id"))
-        val nodeaddress = LightningMessageCodecs.nodeaddress.decode(BitVector(rs.getBytes("data"))).require.value match {
+        val nodeaddress = LightningMessageCodecs.nodeaddress.decode(BitVector(rs.getBytes("data"))).require.value
-          case IPv4(ipv4, port) => new InetSocketAddress(ipv4, port)
-          case IPv6(ipv6, port) => new InetSocketAddress(ipv6, port)
-          case _ => ???
-        }
         m += (nodeid -> nodeaddress)
       }
       m

eclair-core/src/main/scala/fr/acinq/eclair/io/Authenticator.scala

@@ -18,15 +18,15 @@ package fr.acinq.eclair.io
 
 import java.net.InetSocketAddress
 
-import akka.actor.{Actor, ActorLogging, ActorRef, DiagnosticActorLogging, OneForOneStrategy, Props, Status, SupervisorStrategy, Terminated}
+import akka.actor.{Actor, ActorRef, DiagnosticActorLogging, OneForOneStrategy, Props, Status, SupervisorStrategy, Terminated}
 import akka.event.Logging.MDC
 import fr.acinq.bitcoin.Crypto.PublicKey
-import fr.acinq.eclair.{Logs, NodeParams}
 import fr.acinq.eclair.crypto.Noise.KeyPair
 import fr.acinq.eclair.crypto.TransportHandler
 import fr.acinq.eclair.crypto.TransportHandler.HandshakeCompleted
 import fr.acinq.eclair.io.Authenticator.{Authenticated, AuthenticationFailed, PendingAuth}
 import fr.acinq.eclair.wire.LightningMessageCodecs
+import fr.acinq.eclair.{Logs, NodeParams}
 
 /**
   * The purpose of this class is to serve as a buffer for newly connection before they are authenticated

eclair-core/src/main/scala/fr/acinq/eclair/io/Client.scala

@@ -23,8 +23,11 @@ import akka.event.Logging.MDC
 import akka.io.Tcp.SO.KeepAlive
 import akka.io.{IO, Tcp}
 import fr.acinq.bitcoin.Crypto.PublicKey
-import fr.acinq.eclair.{Logs, NodeParams}
 import fr.acinq.eclair.io.Client.ConnectionFailed
+import fr.acinq.eclair.tor.Socks5Connection.{Socks5Connect, Socks5Connected}
+import fr.acinq.eclair.tor.{Socks5Connection, Socks5ProxyParams}
+import fr.acinq.eclair.wire.NodeAddress
+import fr.acinq.eclair.{Logs, NodeParams}
 
 import scala.concurrent.duration._
 
@@ -32,46 +35,83 @@ import scala.concurrent.duration._
   * Created by PM on 27/10/2015.
   *
   */
-class Client(nodeParams: NodeParams, authenticator: ActorRef, address: InetSocketAddress, remoteNodeId: PublicKey, origin_opt: Option[ActorRef]) extends Actor with DiagnosticActorLogging {
+class Client(nodeParams: NodeParams, authenticator: ActorRef, remoteAddress: InetSocketAddress, remoteNodeId: PublicKey, origin_opt: Option[ActorRef]) extends Actor with DiagnosticActorLogging {
 
-  import Tcp._
   import context.system
 
   // we could connect directly here but this allows to take advantage of the automated mdc configuration on message reception
   self ! 'connect
 
-  def receive = {
+  def receive: Receive = {
     case 'connect =>
-      log.info(s"connecting to pubkey=$remoteNodeId host=${address.getHostString} port=${address.getPort}")
+      val (peerOrProxyAddress, proxyParams_opt) = nodeParams.socksProxy_opt.map(proxyParams => (proxyParams, Socks5ProxyParams.proxyAddress(remoteAddress, proxyParams))) match {
-      IO(Tcp) ! Connect(address, timeout = Some(5 seconds), options = KeepAlive(true) :: Nil, pullMode = true)
+          case Some((proxyParams, Some(proxyAddress))) =>
+          log.info(s"connecting to SOCKS5 proxy ${str(proxyAddress)}")
+          (proxyAddress, Some(proxyParams))
+        case _ =>
+          log.info(s"connecting to ${str(remoteAddress)}")
+          (remoteAddress, None)
+      }
+      IO(Tcp) ! Tcp.Connect(peerOrProxyAddress, timeout = Some(50 seconds), options = KeepAlive(true) :: Nil, pullMode = true)
+      context become connecting(proxyParams_opt)
+  }
 
-    case CommandFailed(_: Connect) =>
+  def connecting(proxyParams: Option[Socks5ProxyParams]): Receive = {
-      log.info(s"connection failed to $remoteNodeId@${address.getHostString}:${address.getPort}")
+    case Tcp.CommandFailed(c: Tcp.Connect) =>
-      origin_opt.map(_ ! Status.Failure(ConnectionFailed(address)))
+      val peerOrProxyAddress = c.remoteAddress
+      log.info(s"connection failed to ${str(peerOrProxyAddress)}")
+      origin_opt.map(_ ! Status.Failure(ConnectionFailed(remoteAddress)))
       context stop self
 
-    case Connected(remote, _) =>
+    case Tcp.Connected(peerOrProxyAddress, _) =>
-      log.info(s"connected to pubkey=$remoteNodeId host=${remote.getHostString} port=${remote.getPort}")
+      val connection = sender()
-      val connection = sender
-      authenticator ! Authenticator.PendingAuth(connection, remoteNodeId_opt = Some(remoteNodeId), address = address, origin_opt = origin_opt)
       context watch connection
+      proxyParams match {
+        case Some(proxyParams) =>
+          val proxyAddress = peerOrProxyAddress
+          log.info(s"connected to SOCKS5 proxy ${str(proxyAddress)}")
+          log.info(s"connecting to ${str(remoteAddress)} via SOCKS5 ${str(proxyAddress)}")
+          val proxy = context.actorOf(Socks5Connection.props(sender(), Socks5ProxyParams.proxyCredentials(proxyParams), Socks5Connect(remoteAddress)))
+          context become {
+            case Tcp.CommandFailed(_: Socks5Connect) =>
+              log.info(s"connection failed to ${str(remoteAddress)} via SOCKS5 ${str(proxyAddress)}")
+              origin_opt.map(_ ! Status.Failure(ConnectionFailed(remoteAddress)))
+              context stop self
+            case Socks5Connected(_) =>
+              log.info(s"connected to ${str(remoteAddress)} via SOCKS5 proxy ${str(proxyAddress)}")
+              auth(proxy)
+              context become connected(proxy)
+          }
+        case None =>
+          val peerAddress = peerOrProxyAddress
+          log.info(s"connected to ${str(peerAddress)}")
+          auth(connection)
           context become connected(connection)
       }
+  }
 
   def connected(connection: ActorRef): Receive = {
     case Terminated(actor) if actor == connection =>
       context stop self
   }
 
-  override def unhandled(message: Any): Unit = log.warning(s"unhandled message=$message")
+  override def unhandled(message: Any): Unit = {
-
+    log.warning(s"unhandled message=$message")
-  override def mdc(currentMessage: Any): MDC = Logs.mdc(remoteNodeId_opt = Some(remoteNodeId))
   }
 
-object Client extends App {
+  // we should not restart a failing socks client
+  override val supervisorStrategy = OneForOneStrategy(loggingEnabled = true) { case _ => SupervisorStrategy.Stop }
+
+  override def mdc(currentMessage: Any): MDC = Logs.mdc(remoteNodeId_opt = Some(remoteNodeId))
+
+  private def str(address: InetSocketAddress): String = s"${address.getHostString}:${address.getPort}"
+
+  def auth(connection: ActorRef) = authenticator ! Authenticator.PendingAuth(connection, remoteNodeId_opt = Some(remoteNodeId), address = remoteAddress, origin_opt = origin_opt)
+}
+
+object Client {
 
   def props(nodeParams: NodeParams, authenticator: ActorRef, address: InetSocketAddress, remoteNodeId: PublicKey, origin_opt: Option[ActorRef]): Props = Props(new Client(nodeParams, authenticator, address, remoteNodeId, origin_opt))
 
   case class ConnectionFailed(address: InetSocketAddress) extends RuntimeException(s"connection failed to $address")
-
 }

eclair-core/src/main/scala/fr/acinq/eclair/io/Peer.scala

@@ -21,6 +21,7 @@ import java.net.InetSocketAddress
 import java.nio.ByteOrder
 
 import akka.actor.{Actor, ActorRef, OneForOneStrategy, PoisonPill, Props, Status, SupervisorStrategy, Terminated}
+import akka.actor.{ActorRef, FSM, OneForOneStrategy, PoisonPill, Props, Status, SupervisorStrategy, Terminated}
 import akka.event.Logging.MDC
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.bitcoin.{BinaryData, DeterministicWallet, MilliSatoshi, Protocol, Satoshi}
@@ -56,20 +57,27 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
   }
 
   when(DISCONNECTED) {
-    case Event(Peer.Connect(NodeURI(_, address)), _) =>
+    case Event(Peer.Connect(NodeURI(_, hostAndPort)), d: DisconnectedData) =>
-      // even if we are in a reconnection loop, we immediately process explicit connection requests
+      val address = new InetSocketAddress(hostAndPort.getHost, hostAndPort.getPort)
-      context.actorOf(Client.props(nodeParams, authenticator, new InetSocketAddress(address.getHost, address.getPort), remoteNodeId, origin_opt = Some(sender())))
+      if (d.address_opt.contains(address)) {
+        // we already know this address, we'll reconnect automatically
+        sender ! "reconnection in progress"
         stay
+      } else {
+        // we immediately process explicit connection requests to new addresses
+        context.actorOf(Client.props(nodeParams, authenticator, address, remoteNodeId, origin_opt = Some(sender())))
+        stay
+      }
 
-    case Event(Reconnect, d@DisconnectedData(address_opt, channels, attempts)) =>
+    case Event(Reconnect, d: DisconnectedData) =>
-      address_opt match {
+      d.address_opt match {
         case None => stay // no-op (this peer didn't initiate the connection and doesn't have the ip of the counterparty)
-        case _ if channels.isEmpty => stay // no-op (no more channels with this peer)
+        case _ if d.channels.isEmpty => stay // no-op (no more channels with this peer)
         case Some(address) =>
           context.actorOf(Client.props(nodeParams, authenticator, address, remoteNodeId, origin_opt = None))
           // exponential backoff retry with a finite max
-          setTimer(RECONNECT_TIMER, Reconnect, Math.min(10 + Math.pow(2, attempts), 60) seconds, repeat = false)
+          setTimer(RECONNECT_TIMER, Reconnect, Math.min(10 + Math.pow(2, d.attempts), 60) seconds, repeat = false)
-          stay using d.copy(attempts = attempts + 1)
+          stay using d.copy(attempts = d.attempts + 1)
       }
 
     case Event(Authenticator.Authenticated(_, transport, remoteNodeId1, address, outgoing, origin_opt), d: DisconnectedData) =>
@@ -84,16 +92,25 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
       log.info(s"using globalFeatures=${localInit.globalFeatures} and localFeatures=${localInit.localFeatures}")
       transport ! localInit
 
-      // we store the ip upon successful outgoing connection, keeping only the most recent one
+      val address_opt = if (outgoing) {
-      if (outgoing) {
+        // we store the node address upon successful outgoing connection, so we can reconnect later
-        nodeParams.peersDb.addOrUpdatePeer(remoteNodeId, address)
+        // any previous address is overwritten
-      }
+        NodeAddress.fromParts(address.getHostString, address.getPort).map(nodeAddress => nodeParams.peersDb.addOrUpdatePeer(remoteNodeId, nodeAddress))
-      goto(INITIALIZING) using InitializingData(if (outgoing) Some(address) else None, transport, d.channels, origin_opt, localInit)
+        Some(address)
+      } else None
 
-    case Event(Terminated(actor), d@DisconnectedData(_, channels, _)) if channels.exists(_._2 == actor) =>
+      goto(INITIALIZING) using InitializingData(address_opt, transport, d.channels, origin_opt, localInit)
-      val h = channels.filter(_._2 == actor).keys
+
+    case Event(Terminated(actor), d: DisconnectedData) if d.channels.exists(_._2 == actor) =>
+      val h = d.channels.filter(_._2 == actor).keys
       log.info(s"channel closed: channelId=${h.mkString("/")}")
-      stay using d.copy(channels = channels -- h)
+      val channels1 = d.channels -- h
+      if (channels1.isEmpty) {
+        // we have no existing channels, we can forget about this peer
+        stopPeer()
+      } else {
+        stay using d.copy(channels = channels1)
+      }
 
     case Event(_: wire.LightningMessage, _) => stay // we probably just got disconnected and that's the last messages we received
   }
@@ -153,7 +170,13 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
     case Event(Terminated(actor), d: InitializingData) if d.channels.exists(_._2 == actor) =>
       val h = d.channels.filter(_._2 == actor).keys
       log.info(s"channel closed: channelId=${h.mkString("/")}")
-      stay using d.copy(channels = d.channels -- h)
+      val channels1 = d.channels -- h
+      if (channels1.isEmpty) {
+        // we have no existing channels, we can forget about this peer
+        stopPeer()
+      } else {
+        stay using d.copy(channels = channels1)
+      }
   }
 
   when(CONNECTED) {
@@ -356,8 +379,16 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
           }
           log.error(s"peer sent us a routing message with invalid sig: r=$r bin=$bin")
           // for now we just return an error, maybe ban the peer in the future?
+          // TODO: this doesn't actually disconnect the peer, once we introduce peer banning we should actively disconnect
           d.transport ! Error(CHANNELID_ZERO, s"bad announcement sig! bin=$bin".getBytes())
           d.behavior
+        case InvalidAnnouncement(c) =>
+          // they seem to be sending us fake announcements?
+          log.error(s"couldn't find funding tx with valid scripts for shortChannelId=${c.shortChannelId}")
+          // for now we just return an error, maybe ban the peer in the future?
+          // TODO: this doesn't actually disconnect the peer, once we introduce peer banning we should actively disconnect
+          d.transport ! Error(CHANNELID_ZERO, s"couldn't verify channel! shortChannelId=${c.shortChannelId}".getBytes())
+          d.behavior
         case ChannelClosed(_) =>
           if (d.behavior.ignoreNetworkAnnouncement) {
             // we already are ignoring announcements, we may have additional notifications for announcements that were received right before our ban
@@ -369,18 +400,6 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
             setTimer(ResumeAnnouncements.toString, ResumeAnnouncements, IGNORE_NETWORK_ANNOUNCEMENTS_PERIOD, repeat = false)
             d.behavior.copy(fundingTxAlreadySpentCount = d.behavior.fundingTxAlreadySpentCount + 1, ignoreNetworkAnnouncement = true)
           }
-        case NonexistingChannel(_) =>
-          // this should never happen, unless we are not in sync or there is a 6+ blocks reorg
-          if (d.behavior.ignoreNetworkAnnouncement) {
-            // we already are ignoring announcements, we may have additional notifications for announcements that were received right before our ban
-            d.behavior.copy(fundingTxNotFoundCount = d.behavior.fundingTxNotFoundCount + 1)
-          } else if (d.behavior.fundingTxNotFoundCount < MAX_FUNDING_TX_NOT_FOUND) {
-            d.behavior.copy(fundingTxNotFoundCount = d.behavior.fundingTxNotFoundCount + 1)
-          } else {
-            log.warning(s"peer sent us too many channel announcements with non-existing funding tx (count=${d.behavior.fundingTxNotFoundCount + 1}), ignoring network announcements for $IGNORE_NETWORK_ANNOUNCEMENTS_PERIOD")
-            setTimer(ResumeAnnouncements.toString, ResumeAnnouncements, IGNORE_NETWORK_ANNOUNCEMENTS_PERIOD, repeat = false)
-            d.behavior.copy(fundingTxNotFoundCount = d.behavior.fundingTxNotFoundCount + 1, ignoreNetworkAnnouncement = true)
-          }
       }
       stay using d.copy(behavior = behavior1)
 
@@ -394,8 +413,13 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
 
     case Event(Terminated(actor), d: ConnectedData) if actor == d.transport =>
       log.info(s"lost connection to $remoteNodeId")
+      if (d.channels.isEmpty) {
+        // we have no existing channels, we can forget about this peer
+        stopPeer()
+      } else {
         d.channels.values.toSet[ActorRef].foreach(_ ! INPUT_DISCONNECTED) // we deduplicate with toSet because there might be two entries per channel (tmp id and final id)
         goto(DISCONNECTED) using DisconnectedData(d.address_opt, d.channels.collect { case (k: FinalChannelId, v) => (k, v) })
+      }
 
     case Event(Terminated(actor), d: ConnectedData) if d.channels.values.toSet.contains(actor) =>
       // we will have at most 2 ids: a TemporaryChannelId and a FinalChannelId
@@ -435,16 +459,17 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
 
     case Event(_: TransportHandler.ReadAck, _) => stay // ignored
 
+    case Event(Peer.Reconnect, _) => stay // we got connected in the meantime
+
     case Event(SendPing, _) => stay // we got disconnected in the meantime
 
     case Event(_: Pong, _) => stay // we got disconnected before receiving the pong
 
     case Event(_: PingTimeout, _) => stay // we got disconnected after sending a ping
-
-    case Event(_: Terminated, _) => stay // this channel got closed before having a commitment and we got disconnected (e.g. a funding error occured)
   }
 
   onTransition {
+    case INSTANTIATING -> DISCONNECTED if nodeParams.autoReconnect && nextStateData.address_opt.isDefined => self ! Reconnect // we reconnect right away if we just started the peer
     case _ -> DISCONNECTED if nodeParams.autoReconnect && nextStateData.address_opt.isDefined => setTimer(RECONNECT_TIMER, Reconnect, 1 second, repeat = false)
     case DISCONNECTED -> _ if nodeParams.autoReconnect && stateData.address_opt.isDefined => cancelTimer(RECONNECT_TIMER)
   }
@@ -462,6 +487,12 @@ class Peer(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: Actor
     channel
   }
 
+  def stopPeer() = {
+    log.info("removing peer from db")
+    nodeParams.peersDb.removePeer(remoteNodeId)
+    stop(FSM.Normal)
+  }
+
   // a failing channel won't be restarted, it should handle its states
   override val supervisorStrategy = OneForOneStrategy(loggingEnabled = true) { case _ => SupervisorStrategy.Stop }
 
@@ -497,7 +528,7 @@ object Peer {
 
   val IGNORE_NETWORK_ANNOUNCEMENTS_PERIOD = 5 minutes
 
-  def props(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: ActorRef, watcher: ActorRef, router: ActorRef, relayer: ActorRef, wallet: EclairWallet) = Props(new Peer(nodeParams, remoteNodeId, authenticator, watcher, router, relayer, wallet: EclairWallet))
+  def props(nodeParams: NodeParams, remoteNodeId: PublicKey, authenticator: ActorRef, watcher: ActorRef, router: ActorRef, relayer: ActorRef, wallet: EclairWallet) = Props(new Peer(nodeParams, remoteNodeId, authenticator, watcher, router, relayer, wallet))
 
   // @formatter:off
 
@@ -542,8 +573,8 @@ object Peer {
 
   sealed trait BadMessage
   case class InvalidSignature(r: RoutingMessage) extends BadMessage
+  case class InvalidAnnouncement(c: ChannelAnnouncement) extends BadMessage
   case class ChannelClosed(c: ChannelAnnouncement) extends BadMessage
-  case class NonexistingChannel(c: ChannelAnnouncement) extends BadMessage
 
   case class Behavior(fundingTxAlreadySpentCount: Int = 0, fundingTxNotFoundCount: Int = 0, ignoreNetworkAnnouncement: Boolean = false)
 

eclair-core/src/main/scala/fr/acinq/eclair/io/Server.scala

@@ -18,6 +18,7 @@ package fr.acinq.eclair.io
 
 import java.net.InetSocketAddress
 
+import akka.Done
 import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props, SupervisorStrategy}
 import akka.io.Tcp.SO.KeepAlive
 import akka.io.{IO, Tcp}
@@ -32,7 +33,7 @@ import scala.concurrent.Promise
 /**
   * Created by PM on 27/10/2015.
   */
-class Server(nodeParams: NodeParams, authenticator: ActorRef, address: InetSocketAddress, bound: Option[Promise[Unit]] = None) extends Actor with ActorLogging {
+class Server(nodeParams: NodeParams, authenticator: ActorRef, address: InetSocketAddress, bound: Option[Promise[Done]] = None) extends Actor with ActorLogging {
 
   import Tcp._
   import context.system
@@ -41,7 +42,7 @@ class Server(nodeParams: NodeParams, authenticator: ActorRef, address: InetSocke
 
   def receive() = {
     case Bound(localAddress) =>
-      bound.map(_.success(Unit))
+      bound.map(_.success(Done))
       log.info(s"bound on $localAddress")
       // Accept connections one by one
       sender() ! ResumeAccepting(batchSize = 1)
@@ -65,7 +66,7 @@ class Server(nodeParams: NodeParams, authenticator: ActorRef, address: InetSocke
 
 object Server {
 
-  def props(nodeParams: NodeParams, switchboard: ActorRef, address: InetSocketAddress, bound: Option[Promise[Unit]] = None): Props = Props(new Server(nodeParams, switchboard, address, bound))
+  def props(nodeParams: NodeParams, switchboard: ActorRef, address: InetSocketAddress, bound: Option[Promise[Done]] = None): Props = Props(new Server(nodeParams, switchboard, address, bound))
 
 }
 

eclair-core/src/main/scala/fr/acinq/eclair/io/Switchboard.scala

@@ -18,11 +18,11 @@ package fr.acinq.eclair.io
 
 import java.net.InetSocketAddress
 
-import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props, Status, SupervisorStrategy, Terminated}
+import akka.actor.{Actor, ActorLogging, ActorRef, OneForOneStrategy, Props, Status, SupervisorStrategy}
 import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
 import fr.acinq.eclair.NodeParams
 import fr.acinq.eclair.blockchain.EclairWallet
-import fr.acinq.eclair.channel._
+import fr.acinq.eclair.channel.{HasCommitments, _}
 import fr.acinq.eclair.payment.Relayer.RelayPayload
 import fr.acinq.eclair.payment.{Relayed, Relayer}
 import fr.acinq.eclair.router.Rebroadcast
@@ -43,7 +43,7 @@ class Switchboard(nodeParams: NodeParams, authenticator: ActorRef, watcher: Acto
   authenticator ! self
 
   // we load peers and channels from database
-  private val initialPeers = {
+  {
     val channels = nodeParams.channelsDb.listChannels()
     val peers = nodeParams.peersDb.listPeers()
 
@@ -59,69 +59,70 @@ class Switchboard(nodeParams: NodeParams, authenticator: ActorRef, watcher: Acto
       .map {
         case (remoteNodeId, states) => (remoteNodeId, states, peers.get(remoteNodeId))
       }
-      .map {
+      .foreach {
-        case (remoteNodeId, states, address_opt) =>
+        case (remoteNodeId, states, nodeaddress_opt) =>
           // we might not have an address if we didn't initiate the connection in the first place
-          val peer = createOrGetPeer(Map(), remoteNodeId, previousKnownAddress = address_opt, offlineChannels = states.toSet)
+          val address_opt = nodeaddress_opt.map(_.socketAddress)
-          remoteNodeId -> peer
+          createOrGetPeer(remoteNodeId, previousKnownAddress = address_opt, offlineChannels = states.toSet)
-      }.toMap
+      }
   }
 
-  def receive: Receive = main(initialPeers)
+  def receive: Receive = {
-
-  def main(peers: Map[PublicKey, ActorRef]): Receive = {
 
     case Peer.Connect(NodeURI(publicKey, _)) if publicKey == nodeParams.nodeId =>
       sender ! Status.Failure(new RuntimeException("cannot open connection with oneself"))
 
     case c@Peer.Connect(NodeURI(remoteNodeId, _)) =>
       // we create a peer if it doesn't exist
-      val peer = createOrGetPeer(peers, remoteNodeId, previousKnownAddress = None, offlineChannels = Set.empty)
+      val peer = createOrGetPeer(remoteNodeId, previousKnownAddress = None, offlineChannels = Set.empty)
       peer forward c
-      context become main(peers + (remoteNodeId -> peer))
 
     case o@Peer.OpenChannel(remoteNodeId, _, _, _, _) =>
-      peers.get(remoteNodeId) match {
+      getPeer(remoteNodeId) match {
         case Some(peer) => peer forward o
         case None => sender ! Status.Failure(new RuntimeException("no connection to peer"))
       }
 
-    case Terminated(actor) =>
-      peers.collectFirst {
-        case (remoteNodeId, peer) if peer == actor =>
-          log.debug(s"$actor is dead, removing from peers")
-          nodeParams.peersDb.removePeer(remoteNodeId)
-          context become main(peers - remoteNodeId)
-      }
-
     case auth@Authenticator.Authenticated(_, _, remoteNodeId, _, _, _) =>
       // if this is an incoming connection, we might not yet have created the peer
-      val peer = createOrGetPeer(peers, remoteNodeId, previousKnownAddress = None, offlineChannels = Set.empty)
+      val peer = createOrGetPeer(remoteNodeId, previousKnownAddress = None, offlineChannels = Set.empty)
       peer forward auth
-      context become main(peers + (remoteNodeId -> peer))
 
-    case r: Rebroadcast => peers.values.foreach(_ forward r)
+    case r: Rebroadcast => context.children.foreach(_ forward r)
 
-    case 'peers => sender ! peers
+    case 'peers => sender ! context.children
 
   }
 
+  def peerActorName(remoteNodeId: PublicKey): String = s"peer-$remoteNodeId"
+
+  /**
+    * Retrieves a peer based on its public key.
+    *
+    * NB: Internally akka uses a TreeMap to store the binding, so this lookup is O(log(N)) where N is the number of
+    * peers. We could make it O(1) by using our own HashMap, but it creates other problems when we need to remove an
+    * existing peer. This seems like a reasonable trade-off because we only make this call once per connection, and N
+    * should never be very big anyway.
+    *
+    * @param remoteNodeId
+    * @return
+    */
+  def getPeer(remoteNodeId: PublicKey): Option[ActorRef] = context.child(peerActorName(remoteNodeId))
+
   /**
     *
-    * @param peers
     * @param remoteNodeId
     * @param previousKnownAddress only to be set if we know for sure that this ip worked in the past
     * @param offlineChannels
     * @return
     */
-  def createOrGetPeer(peers: Map[PublicKey, ActorRef], remoteNodeId: PublicKey, previousKnownAddress: Option[InetSocketAddress], offlineChannels: Set[HasCommitments]) = {
+  def createOrGetPeer(remoteNodeId: PublicKey, previousKnownAddress: Option[InetSocketAddress], offlineChannels: Set[HasCommitments]) = {
-    peers.get(remoteNodeId) match {
+    getPeer(remoteNodeId) match {
       case Some(peer) => peer
       case None =>
-        log.info(s"creating new peer current=${peers.size}")
+        log.info(s"creating new peer current=${context.children.size}")
-        val peer = context.actorOf(Peer.props(nodeParams, remoteNodeId, authenticator, watcher, router, relayer, wallet), name = s"peer-$remoteNodeId")
+        val peer = context.actorOf(Peer.props(nodeParams, remoteNodeId, authenticator, watcher, router, relayer, wallet), name = peerActorName(remoteNodeId))
         peer ! Peer.Init(previousKnownAddress, offlineChannels)
-        context watch (peer)
         peer
     }
   }

eclair-core/src/main/scala/fr/acinq/eclair/payment/Auditor.scala

@@ -17,8 +17,13 @@
 package fr.acinq.eclair.payment
 
 import akka.actor.{Actor, ActorLogging, Props}
+import fr.acinq.bitcoin.BinaryData
 import fr.acinq.eclair.NodeParams
-import fr.acinq.eclair.channel.NetworkFeePaid
+import fr.acinq.eclair.channel._
+import fr.acinq.eclair.db.{AuditDb, ChannelLifecycleEvent}
+
+import scala.concurrent.duration._
+import scala.concurrent.ExecutionContext
 
 class Auditor(nodeParams: NodeParams) extends Actor with ActorLogging {
 
@@ -26,6 +31,11 @@ class Auditor(nodeParams: NodeParams) extends Actor with ActorLogging {
 
   context.system.eventStream.subscribe(self, classOf[PaymentEvent])
   context.system.eventStream.subscribe(self, classOf[NetworkFeePaid])
+  context.system.eventStream.subscribe(self, classOf[AvailableBalanceChanged])
+  context.system.eventStream.subscribe(self, classOf[ChannelStateChanged])
+  context.system.eventStream.subscribe(self, classOf[ChannelClosed])
+
+  val balanceEventThrottler = context.actorOf(Props(new BalanceEventThrottler(db)))
 
   override def receive: Receive = {
 
@@ -37,11 +47,72 @@ class Auditor(nodeParams: NodeParams) extends Actor with ActorLogging {
 
     case e: NetworkFeePaid => db.add(e)
 
+    case e: AvailableBalanceChanged => balanceEventThrottler ! e
+
+    case e: ChannelStateChanged =>
+      e match {
+        case ChannelStateChanged(_, _, remoteNodeId, WAIT_FOR_FUNDING_LOCKED, NORMAL, d: DATA_NORMAL) =>
+          db.add(ChannelLifecycleEvent(d.channelId, remoteNodeId, d.commitments.commitInput.txOut.amount.toLong, d.commitments.localParams.isFunder, !d.commitments.announceChannel, "created"))
+        case _ => ()
+      }
+
+    case e: ChannelClosed =>
+      db.add(ChannelLifecycleEvent(e.channelId, e.commitments.remoteParams.nodeId, e.commitments.commitInput.txOut.amount.toLong, e.commitments.localParams.isFunder, !e.commitments.announceChannel, e.closeType))
+
   }
 
   override def unhandled(message: Any): Unit = log.warning(s"unhandled msg=$message")
 }
 
+/**
+  * We don't want to log every tiny payment, and we don't want to log probing events.
+  */
+class BalanceEventThrottler(db: AuditDb) extends Actor with ActorLogging {
+
+  import ExecutionContext.Implicits.global
+
+  val delay = 30 seconds
+
+  case class BalanceUpdate(first: AvailableBalanceChanged, last: AvailableBalanceChanged)
+
+  case class ProcessEvent(channelId: BinaryData)
+
+  override def receive: Receive = run(Map.empty)
+
+  def run(pending: Map[BinaryData, BalanceUpdate]): Receive = {
+
+    case e: AvailableBalanceChanged =>
+      pending.get(e.channelId) match {
+        case None =>
+          // we delay the processing of the event in order to smooth variations
+          log.info(s"will log balance event in $delay for channelId=${e.channelId}")
+          context.system.scheduler.scheduleOnce(delay, self, ProcessEvent(e.channelId))
+          context.become(run(pending + (e.channelId -> (BalanceUpdate(e, e)))))
+        case Some(BalanceUpdate(first, _)) =>
+          // we already are about to log a balance event, let's update the data we have
+          log.info(s"updating balance data for channelId=${e.channelId}")
+          context.become(run(pending + (e.channelId -> (BalanceUpdate(first, e)))))
+      }
+
+    case ProcessEvent(channelId) =>
+      pending.get(channelId) match {
+        case Some(BalanceUpdate(first, last)) =>
+          if (first.commitments.remoteCommit.spec.toRemoteMsat == last.localBalanceMsat) {
+            // we don't log anything if the balance didn't change (e.g. it was a probe payment)
+            log.info(s"ignoring balance event for channelId=$channelId (changed was discarded)")
+          } else {
+            log.info(s"processing balance event for channelId=$channelId balance=${first.localBalanceMsat}->${last.localBalanceMsat}")
+            // we log the last event, which contains the most up to date balance
+            db.add(last)
+            context.become(run(pending - channelId))
+          }
+        case None => () // wtf?
+      }
+
+  }
+
+}
+
 object Auditor {
 
   def props(nodeParams: NodeParams) = Props(classOf[Auditor], nodeParams)

eclair-core/src/main/scala/fr/acinq/eclair/payment/Autoprobe.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.payment
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Props}

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentLifecycle.scala

@@ -43,7 +43,7 @@ class PaymentLifecycle(sourceNodeId: PublicKey, router: ActorRef, register: Acto
 
   when(WAITING_FOR_REQUEST) {
     case Event(c: SendPayment, WaitingForRequest) =>
-      router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes)
+      router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, randomize = c.randomize)
       goto(WAITING_FOR_ROUTE) using WaitingForRoute(sender, c, failures = Nil)
   }
 
@@ -103,12 +103,12 @@ class PaymentLifecycle(sourceNodeId: PublicKey, router: ActorRef, register: Acto
           // in that case we don't know which node is sending garbage, let's try to blacklist all nodes except the one we are directly connected to and the destination node
           val blacklist = hops.map(_.nextNodeId).drop(1).dropRight(1)
           log.warning(s"blacklisting intermediate nodes=${blacklist.mkString(",")}")
-          router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes ++ blacklist, ignoreChannels)
+          router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes ++ blacklist, ignoreChannels, c.randomize)
           goto(WAITING_FOR_ROUTE) using WaitingForRoute(s, c, failures :+ UnreadableRemoteFailure(hops))
         case Success(e@ErrorPacket(nodeId, failureMessage: Node)) =>
           log.info(s"received 'Node' type error message from nodeId=$nodeId, trying to route around it (failure=$failureMessage)")
           // let's try to route around this node
-          router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes + nodeId, ignoreChannels)
+          router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes + nodeId, ignoreChannels, c.randomize)
           goto(WAITING_FOR_ROUTE) using WaitingForRoute(s, c, failures :+ RemoteFailure(hops, e))
         case Success(e@ErrorPacket(nodeId, failureMessage: Update)) =>
           log.info(s"received 'Update' type error message from nodeId=$nodeId, retrying payment (failure=$failureMessage)")
@@ -136,18 +136,18 @@ class PaymentLifecycle(sourceNodeId: PublicKey, router: ActorRef, register: Acto
             // in any case, we forward the update to the router
             router ! failureMessage.update
             // let's try again, router will have updated its state
-            router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes, ignoreChannels)
+            router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes, ignoreChannels, c.randomize)
           } else {
             // this node is fishy, it gave us a bad sig!! let's filter it out
             log.warning(s"got bad signature from node=$nodeId update=${failureMessage.update}")
-            router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes + nodeId, ignoreChannels)
+            router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes + nodeId, ignoreChannels, c.randomize)
           }
           goto(WAITING_FOR_ROUTE) using WaitingForRoute(s, c, failures :+ RemoteFailure(hops, e))
         case Success(e@ErrorPacket(nodeId, failureMessage)) =>
           log.info(s"received an error message from nodeId=$nodeId, trying to use a different channel (failure=$failureMessage)")
           // let's try again without the channel outgoing from nodeId
           val faultyChannel = hops.find(_.nodeId == nodeId).map(hop => ChannelDesc(hop.lastUpdate.shortChannelId, hop.nodeId, hop.nextNodeId))
-          router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes, ignoreChannels ++ faultyChannel.toSet)
+          router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes, ignoreChannels ++ faultyChannel.toSet, c.randomize)
           goto(WAITING_FOR_ROUTE) using WaitingForRoute(s, c, failures :+ RemoteFailure(hops, e))
       }
 
@@ -166,7 +166,7 @@ class PaymentLifecycle(sourceNodeId: PublicKey, router: ActorRef, register: Acto
       } else {
         log.info(s"received an error message from local, trying to use a different channel (failure=${t.getMessage})")
         val faultyChannel = ChannelDesc(hops.head.lastUpdate.shortChannelId, hops.head.nodeId, hops.head.nextNodeId)
-        router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes, ignoreChannels + faultyChannel)
+        router ! RouteRequest(sourceNodeId, c.targetNodeId, c.amountMsat, c.assistedRoutes, ignoreNodes, ignoreChannels + faultyChannel, c.randomize)
         goto(WAITING_FOR_ROUTE) using WaitingForRoute(s, c, failures :+ LocalFailure(t))
       }
 
@@ -193,7 +193,7 @@ object PaymentLifecycle {
   /**
     * @param maxFeePct set by default to 3% as a safety measure (even if a route is found, if fee is higher than that payment won't be attempted)
     */
-  case class SendPayment(amountMsat: Long, paymentHash: BinaryData, targetNodeId: PublicKey, assistedRoutes: Seq[Seq[ExtraHop]] = Nil, finalCltvExpiry: Long = Channel.MIN_CLTV_EXPIRY, maxAttempts: Int = 5, maxFeePct: Double = 0.03) {
+  case class SendPayment(amountMsat: Long, paymentHash: BinaryData, targetNodeId: PublicKey, assistedRoutes: Seq[Seq[ExtraHop]] = Nil, finalCltvExpiry: Long = Channel.MIN_CLTV_EXPIRY, maxAttempts: Int = 5, maxFeePct: Double = 0.03, randomize: Option[Boolean] = None) {
     require(amountMsat > 0, s"amountMsat must be > 0")
   }
   case class CheckPayment(paymentHash: BinaryData)

eclair-core/src/main/scala/fr/acinq/eclair/payment/Relayer.scala

@@ -70,16 +70,15 @@ class Relayer(nodeParams: NodeParams, register: ActorRef, paymentHandler: ActorR
 
     case LocalChannelUpdate(_, channelId, shortChannelId, remoteNodeId, _, channelUpdate, commitments) =>
       log.debug(s"updating local channel info for channelId=$channelId shortChannelId=$shortChannelId remoteNodeId=$remoteNodeId channelUpdate={} commitments={}", channelUpdate, commitments)
-      val availableLocalBalance = commitments.remoteCommit.spec.toRemoteMsat // note that remoteCommit.toRemote == toLocal
+      context become main(channelUpdates + (channelUpdate.shortChannelId -> OutgoingChannel(remoteNodeId, channelUpdate, commitments.availableBalanceForSendMsat)), node2channels.addBinding(remoteNodeId, channelUpdate.shortChannelId))
-      context become main(channelUpdates + (channelUpdate.shortChannelId -> OutgoingChannel(remoteNodeId, channelUpdate, availableLocalBalance)), node2channels.addBinding(remoteNodeId, channelUpdate.shortChannelId))
 
     case LocalChannelDown(_, channelId, shortChannelId, remoteNodeId) =>
       log.debug(s"removed local channel info for channelId=$channelId shortChannelId=$shortChannelId")
       context become main(channelUpdates - shortChannelId, node2channels.removeBinding(remoteNodeId, shortChannelId))
 
-    case AvailableBalanceChanged(_, _, shortChannelId, localBalanceMsat) =>
+    case AvailableBalanceChanged(_, _, shortChannelId, _, commitments) =>
       val channelUpdates1 = channelUpdates.get(shortChannelId) match {
-        case Some(c: OutgoingChannel) => channelUpdates + (shortChannelId -> c.copy(availableBalanceMsat = localBalanceMsat))
+        case Some(c: OutgoingChannel) => channelUpdates + (shortChannelId -> c.copy(availableBalanceMsat = commitments.availableBalanceForSendMsat))
         case None => channelUpdates // we only consider the balance if we have the channel_update
       }
       context become main(channelUpdates1, node2channels)
@@ -200,7 +199,7 @@ object Relayer {
   sealed trait NextPayload
   case class FinalPayload(add: UpdateAddHtlc, payload: PerHopPayload) extends NextPayload
   case class RelayPayload(add: UpdateAddHtlc, payload: PerHopPayload, nextPacket: Sphinx.Packet) extends NextPayload {
-    val relayFeeSatoshi = add.amountMsat - payload.amtToForward
+    val relayFeeMsat = add.amountMsat - payload.amtToForward
     val expiryDelta = add.cltvExpiry - payload.outgoingCltvValue
   }
   // @formatter:on
@@ -264,19 +263,19 @@ object Relayer {
       case Some(channelUpdate) if !Announcements.isEnabled(channelUpdate.channelFlags) =>
         Left(CMD_FAIL_HTLC(add.id, Right(ChannelDisabled(channelUpdate.messageFlags, channelUpdate.channelFlags, channelUpdate)), commit = true))
       case Some(channelUpdate) if payload.amtToForward < channelUpdate.htlcMinimumMsat =>
-        Left(CMD_FAIL_HTLC(add.id, Right(AmountBelowMinimum(add.amountMsat, channelUpdate)), commit = true))
+        Left(CMD_FAIL_HTLC(add.id, Right(AmountBelowMinimum(payload.amtToForward, channelUpdate)), commit = true))
       case Some(channelUpdate) if relayPayload.expiryDelta != channelUpdate.cltvExpiryDelta =>
-        Left(CMD_FAIL_HTLC(add.id, Right(IncorrectCltvExpiry(add.cltvExpiry, channelUpdate)), commit = true))
+        Left(CMD_FAIL_HTLC(add.id, Right(IncorrectCltvExpiry(payload.outgoingCltvValue, channelUpdate)), commit = true))
-      case Some(channelUpdate) if relayPayload.relayFeeSatoshi < nodeFee(channelUpdate.feeBaseMsat, channelUpdate.feeProportionalMillionths, payload.amtToForward) =>
+      case Some(channelUpdate) if relayPayload.relayFeeMsat < nodeFee(channelUpdate.feeBaseMsat, channelUpdate.feeProportionalMillionths, payload.amtToForward) =>
         Left(CMD_FAIL_HTLC(add.id, Right(FeeInsufficient(add.amountMsat, channelUpdate)), commit = true))
       case Some(channelUpdate) =>
-        val isRedirected = (channelUpdate.shortChannelId != payload.shortChannelId) // we may decide to use another channel (to the same node) that the one requested
+        val isRedirected = (channelUpdate.shortChannelId != payload.shortChannelId) // we may decide to use another channel (to the same node) from the one requested
         Right(CMD_ADD_HTLC(payload.amtToForward, add.paymentHash, payload.outgoingCltvValue, nextPacket.serialize, upstream_opt = Some(add), commit = true, redirected = isRedirected))
     }
   }
 
   /**
-    * Select a channel to the same node to the relay the payment to, that has the highest balance and is compatible in
+    * Select a channel to the same node to the relay the payment to, that has the lowest balance and is compatible in
     * terms of fees, expiry_delta, etc.
     *
     * If no suitable channel is found we default to the originally requested channel.
@@ -293,14 +292,13 @@ object Relayer {
     log.debug(s"selecting next channel for htlc #{} paymentHash={} from channelId={} to requestedShortChannelId={}", add.id, add.paymentHash, add.channelId, requestedShortChannelId)
     // first we find out what is the next node
     channelUpdates.get(requestedShortChannelId) match {
-      case Some(OutgoingChannel(nextNodeId, _, requestedChannelId)) =>
+      case Some(OutgoingChannel(nextNodeId, _, _)) =>
         log.debug(s"next hop for htlc #{} paymentHash={} is nodeId={}", add.id, add.paymentHash, nextNodeId)
         // then we retrieve all known channels to this node
-        val candidateChannels = node2channels.get(nextNodeId).getOrElse(Set.empty)
+        val candidateChannels = node2channels.get(nextNodeId).getOrElse(Set.empty[ShortChannelId])
         // and we filter keep the ones that are compatible with this payment (mainly fees, expiry delta)
         candidateChannels
-          .map {
+          .map { shortChannelId =>
-            case shortChannelId =>
               val channelInfo_opt = channelUpdates.get(shortChannelId)
               val channelUpdate_opt = channelInfo_opt.map(_.channelUpdate)
               val relayResult = handleRelay(relayPayload, channelUpdate_opt)
@@ -308,9 +306,10 @@ object Relayer {
               (shortChannelId, channelInfo_opt, relayResult)
           }
           .collect { case (shortChannelId, Some(channelInfo), Right(_)) => (shortChannelId, channelInfo.availableBalanceMsat) }
+          .filter(_._2 > relayPayload.payload.amtToForward) // we only keep channels that have enough balance to handle this payment
           .toList // needed for ordering
-          .sortBy(_._2) // we want to use the channel with the highest available balance
+          .sortBy(_._2) // we want to use the channel with the lowest available balance that can process the payment
-          .lastOption match {
+          .headOption match {
           case Some((preferredShortChannelId, availableBalanceMsat)) if preferredShortChannelId != requestedShortChannelId =>
             log.info("replacing requestedShortChannelId={} by preferredShortChannelId={} with availableBalanceMsat={}", requestedShortChannelId, preferredShortChannelId, availableBalanceMsat)
             preferredShortChannelId

eclair-core/src/main/scala/fr/acinq/eclair/router/Announcements.scala

@@ -20,8 +20,8 @@ import java.net.InetSocketAddress
 
 import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey, sha256, verifySignature}
 import fr.acinq.bitcoin.{BinaryData, Crypto, LexicographicalOrdering}
-import fr.acinq.eclair.{ShortChannelId, serializationResult}
 import fr.acinq.eclair.wire._
+import fr.acinq.eclair.{ShortChannelId, serializationResult}
 import scodec.bits.BitVector
 import shapeless.HNil
 
@@ -75,9 +75,8 @@ object Announcements {
     )
   }
 
-  def makeNodeAnnouncement(nodeSecret: PrivateKey, alias: String, color: Color, addresses: List[InetSocketAddress], timestamp: Long = Platform.currentTime / 1000): NodeAnnouncement = {
+  def makeNodeAnnouncement(nodeSecret: PrivateKey, alias: String, color: Color, nodeAddresses: List[NodeAddress], timestamp: Long = Platform.currentTime / 1000): NodeAnnouncement = {
     require(alias.size <= 32)
-    val nodeAddresses = addresses.map(NodeAddress(_))
     val witness = nodeAnnouncementWitnessEncode(timestamp, nodeSecret.publicKey, color, alias, "", nodeAddresses)
     val sig = Crypto.encodeSignature(Crypto.sign(witness, nodeSecret)) :+ 1.toByte
     NodeAnnouncement(

eclair-core/src/main/scala/fr/acinq/eclair/router/ChannelRangeQueries.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.router
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream}

eclair-core/src/main/scala/fr/acinq/eclair/router/Graph.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.router
 
 import fr.acinq.bitcoin.Crypto.PublicKey

eclair-core/src/main/scala/fr/acinq/eclair/router/Router.scala

@@ -16,6 +16,7 @@
 
 package fr.acinq.eclair.router
 
+import akka.Done
 import akka.actor.{Actor, ActorRef, Props, Status}
 import akka.event.Logging.MDC
 import fr.acinq.bitcoin.Crypto.PublicKey
@@ -24,11 +25,10 @@ import fr.acinq.eclair._
 import fr.acinq.eclair.blockchain._
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.crypto.TransportHandler
-import fr.acinq.eclair.io.Peer.{ChannelClosed, InvalidSignature, NonexistingChannel, PeerRoutingMessage}
+import fr.acinq.eclair.io.Peer.{ChannelClosed, InvalidAnnouncement, InvalidSignature, PeerRoutingMessage}
 import fr.acinq.eclair.payment.PaymentRequest.ExtraHop
 import fr.acinq.eclair.router.Graph.GraphStructure.DirectedGraph.graphEdgeToHop
 import fr.acinq.eclair.router.Graph.GraphStructure.{DirectedGraph, GraphEdge}
-import fr.acinq.eclair.router.Graph.WeightedPath
 import fr.acinq.eclair.transactions.Scripts
 import fr.acinq.eclair.wire._
 
@@ -43,7 +43,7 @@ import scala.util.{Random, Try}
 
 case class ChannelDesc(shortChannelId: ShortChannelId, a: PublicKey, b: PublicKey)
 case class Hop(nodeId: PublicKey, nextNodeId: PublicKey, lastUpdate: ChannelUpdate)
-case class RouteRequest(source: PublicKey, target: PublicKey, amountMsat: Long, assistedRoutes: Seq[Seq[ExtraHop]] = Nil, ignoreNodes: Set[PublicKey] = Set.empty, ignoreChannels: Set[ChannelDesc] = Set.empty)
+case class RouteRequest(source: PublicKey, target: PublicKey, amountMsat: Long, assistedRoutes: Seq[Seq[ExtraHop]] = Nil, ignoreNodes: Set[PublicKey] = Set.empty, ignoreChannels: Set[ChannelDesc] = Set.empty, randomize: Option[Boolean] = None)
 case class RouteResponse(hops: Seq[Hop], ignoreNodes: Set[PublicKey], ignoreChannels: Set[ChannelDesc]) {
   require(hops.size > 0, "route cannot be empty")
 }
@@ -83,7 +83,7 @@ case object TickPruneStaleChannels
   * Created by PM on 24/05/2016.
   */
 
-class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Promise[Unit]] = None) extends FSMDiagnosticActorLogging[State, Data] {
+class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Promise[Done]] = None) extends FSMDiagnosticActorLogging[State, Data] {
 
   import Router._
 
@@ -133,7 +133,7 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
     self ! nodeAnn
 
     log.info(s"initialization completed, ready to process messages")
-    Try(initialized.map(_.success(())))
+    Try(initialized.map(_.success(Done)))
     startWith(NORMAL, Data(initNodes, initChannels, initChannelUpdates, Stash(Map.empty, Map.empty), rebroadcast = Rebroadcast(channels = Map.empty, updates = Map.empty, nodes = Map.empty), awaiting = Map.empty, privateChannels = Map.empty, privateUpdates = Map.empty, excludedChannels = Set.empty, graph, sync = Map.empty))
   }
 
@@ -192,25 +192,26 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
       sender ! RoutingState(d.channels.values, d.updates.values, d.nodes.values)
       stay
 
-    case Event(v@ValidateResult(c, _, _, _), d0) =>
+    case Event(v@ValidateResult(c, _), d0) =>
       d0.awaiting.get(c) match {
         case Some(origin +: others) => origin ! TransportHandler.ReadAck(c) // now we can acknowledge the message, we only need to do it for the first peer that sent us the announcement
         case _ => ()
       }
       log.info("got validation result for shortChannelId={} (awaiting={} stash.nodes={} stash.updates={})", c.shortChannelId, d0.awaiting.size, d0.stash.nodes.size, d0.stash.updates.size)
       val success = v match {
-        case ValidateResult(c, _, _, Some(t)) =>
+        case ValidateResult(c, Left(t)) =>
           log.warning("validation failure for shortChannelId={} reason={}", c.shortChannelId, t.getMessage)
           false
-        case ValidateResult(c, Some(tx), true, None) =>
+        case ValidateResult(c, Right((tx, UtxoStatus.Unspent))) =>
           val TxCoordinates(_, _, outputIndex) = ShortChannelId.coordinates(c.shortChannelId)
           // let's check that the output is indeed a P2WSH multisig 2-of-2 of nodeid1 and nodeid2)
           val fundingOutputScript = write(pay2wsh(Scripts.multiSig2of2(PublicKey(c.bitcoinKey1), PublicKey(c.bitcoinKey2))))
-          if (tx.txOut.size < outputIndex + 1) {
+          if (tx.txOut.size < outputIndex + 1 || fundingOutputScript != tx.txOut(outputIndex).publicKeyScript) {
-            log.error("invalid script for shortChannelId={}: txid={} does not have outputIndex={} ann={}", c.shortChannelId, tx.txid, outputIndex, c)
+            log.error(s"invalid script for shortChannelId={}: txid={} does not have script=$fundingOutputScript at outputIndex=$outputIndex ann={}", c.shortChannelId, tx.txid, c)
-            false
+            d0.awaiting.get(c) match {
-          } else if (fundingOutputScript != tx.txOut(outputIndex).publicKeyScript) {
+              case Some(origins) => origins.foreach(_ ! InvalidAnnouncement(c))
-            log.error("invalid script for shortChannelId={} txid={} ann={}", c.shortChannelId, tx.txid, c)
+              case _ => ()
+            }
             false
           } else {
             watcher ! WatchSpentBasic(self, tx, outputIndex, BITCOIN_FUNDING_EXTERNAL_CHANNEL_SPENT(c.shortChannelId))
@@ -228,23 +229,20 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
             }
             true
           }
-        case ValidateResult(c, Some(tx), false, None) =>
+        case ValidateResult(c, Right((tx, fundingTxStatus: UtxoStatus.Spent))) =>
-          log.warning("ignoring shortChannelId={} tx={} (funding tx already spent)", c.shortChannelId, tx.txid)
+          if (fundingTxStatus.spendingTxConfirmed) {
+            log.warning("ignoring shortChannelId={} tx={} (funding tx already spent and spending tx is confirmed)", c.shortChannelId, tx.txid)
+            // the funding tx has been spent by a transaction that is now confirmed: peer shouldn't send us those
             d0.awaiting.get(c) match {
               case Some(origins) => origins.foreach(_ ! ChannelClosed(c))
               case _ => ()
             }
+          } else {
+            log.debug("ignoring shortChannelId={} tx={} (funding tx already spent but spending tx isn't confirmed)", c.shortChannelId, tx.txid)
+          }
           // there may be a record if we have just restarted
           db.removeChannel(c.shortChannelId)
           false
-        case ValidateResult(c, None, _, None) =>
-          // we couldn't find the funding tx in the blockchain, this is highly suspicious because it should have at least 6 confirmations to be announced
-          log.warning("could not retrieve tx for shortChannelId={}", c.shortChannelId)
-          d0.awaiting.get(c) match {
-            case Some(origins) => origins.foreach(_ ! NonexistingChannel(c))
-            case _ => ()
-          }
-          false
       }
 
       // we also reprocess node and channel_update announcements related to channels that were just analyzed
@@ -373,7 +371,7 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
       sender ! d
       stay
 
-    case Event(RouteRequest(start, end, amount, assistedRoutes, ignoreNodes, ignoreChannels), d) =>
+    case Event(RouteRequest(start, end, amount, assistedRoutes, ignoreNodes, ignoreChannels, randomize), d) =>
       // we convert extra routing info provided in the payment request to fake channel_update
       // it takes precedence over all other channel_updates we know
       val assistedUpdates = assistedRoutes.flatMap(toFakeUpdates(_, end)).toMap
@@ -382,8 +380,9 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
       val ignoredUpdates = getIgnoredChannelDesc(d.updates ++ d.privateUpdates ++ assistedUpdates, ignoreNodes) ++ ignoreChannels ++ d.excludedChannels
       log.info(s"finding a route $start->$end with assistedChannels={} ignoreNodes={} ignoreChannels={} excludedChannels={}", assistedUpdates.keys.mkString(","), ignoreNodes.map(_.toBin).mkString(","), ignoreChannels.mkString(","), d.excludedChannels.mkString(","))
       val extraEdges = assistedUpdates.map { case (c, u) => GraphEdge(c, u) }.toSet
-      // we ask the router to make a random selection among the three best routes, numRoutes = 3
+      // if we want to randomize we ask the router to make a random selection among the three best routes
-      findRoute(d.graph, start, end, amount, numRoutes = DEFAULT_ROUTES_COUNT, extraEdges = extraEdges, ignoredEdges = ignoredUpdates.toSet)
+      val routesToFind = if(randomize.getOrElse(nodeParams.randomizeRouteSelection)) DEFAULT_ROUTES_COUNT else 1
+      findRoute(d.graph, start, end, amount, numRoutes = routesToFind, extraEdges = extraEdges, ignoredEdges = ignoredUpdates.toSet)
         .map(r => sender ! RouteResponse(r, ignoreNodes, ignoreChannels))
         .recover { case t => sender ! Status.Failure(t) }
       stay
@@ -396,8 +395,11 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
       log.info("sending query_channel_range={}", query)
       remote ! query
 
-      // we also set a pass-all filter for now (we can update it later)
+      // we also set a pass-all filter for now (we can update it later) for the future gossip messages, by setting
-      val filter = GossipTimestampFilter(nodeParams.chainHash, firstTimestamp = 0, timestampRange = Int.MaxValue)
+      // the first_timestamp field to the current date/time and timestamp_range to the maximum value
+      // NB: we can't just set firstTimestamp to 0, because in that case peer would send us all past messages matching
+      // that (i.e. the whole routing table)
+      val filter = GossipTimestampFilter(nodeParams.chainHash, firstTimestamp = Platform.currentTime / 1000, timestampRange = Int.MaxValue)
       remote ! filter
 
       // clean our sync state for this peer: we receive a SendChannelQuery just when we connect/reconnect to a peer and
@@ -712,7 +714,7 @@ class Router(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Prom
 
 object Router {
 
-  def props(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Promise[Unit]] = None) = Props(new Router(nodeParams, watcher, initialized))
+  def props(nodeParams: NodeParams, watcher: ActorRef, initialized: Option[Promise[Done]] = None) = Props(new Router(nodeParams, watcher, initialized))
 
   def toFakeUpdate(extraHop: ExtraHop): ChannelUpdate =
   // the `direction` bit in flags will not be accurate but it doesn't matter because it is not used
@@ -832,7 +834,10 @@ object Router {
     val foundRoutes = Graph.yenKshortestPaths(g, localNodeId, targetNodeId, amountMsat, ignoredEdges, extraEdges, numRoutes).toList match {
       case Nil => throw RouteNotFound
       case route :: Nil if route.path.isEmpty => throw RouteNotFound
-      case foundRoutes => foundRoutes
+      case routes => routes.find(_.path.size == 1) match {
+        case Some(directRoute) => directRoute :: Nil
+        case _ => routes
+      }
     }
 
     // minimum cost

eclair-core/src/main/scala/fr/acinq/eclair/tor/Controller.scala

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.tor
+
+import java.net.InetSocketAddress
+
+import akka.actor.{Actor, ActorLogging, OneForOneStrategy, Props, SupervisorStrategy, Terminated}
+import akka.io.{IO, Tcp}
+import akka.util.ByteString
+
+import scala.concurrent.{ExecutionContext, Promise}
+
+/**
+  * Created by rorp
+  *
+  * @param address              Tor control address
+  * @param protocolHandlerProps Tor protocol handler props
+  * @param ec                   execution context
+  */
+class Controller(address: InetSocketAddress, protocolHandlerProps: Props)
+                (implicit ec: ExecutionContext = ExecutionContext.global) extends Actor with ActorLogging {
+
+  import Controller._
+  import Tcp._
+  import context.system
+
+  IO(Tcp) ! Connect(address)
+
+  def receive = {
+    case e@CommandFailed(_: Connect) =>
+      e.cause match {
+        case Some(ex) => log.error(ex, "Cannot connect")
+        case _ => log.error("Cannot connect")
+      }
+      context stop self
+    case c: Connected =>
+      val protocolHandler = context actorOf protocolHandlerProps
+      protocolHandler ! c
+      val connection = sender()
+      connection ! Register(self)
+      context watch connection
+      context become {
+        case data: ByteString =>
+          connection ! Write(data)
+        case CommandFailed(w: Write) =>
+          // O/S buffer was full
+          protocolHandler ! SendFailed
+          log.error("Tor command failed")
+        case Received(data) =>
+          protocolHandler ! data
+        case _: ConnectionClosed =>
+          context stop self
+        case Terminated(actor) if actor == connection =>
+          context stop self
+      }
+  }
+
+  // we should not restart a failing tor session
+  override val supervisorStrategy = OneForOneStrategy(loggingEnabled = true) { case _ => SupervisorStrategy.Escalate }
+
+}
+
+object Controller {
+  def props(address: InetSocketAddress, protocolHandlerProps: Props)(implicit ec: ExecutionContext = ExecutionContext.global) =
+    Props(new Controller(address, protocolHandlerProps))
+
+  case object SendFailed
+
+}

eclair-core/src/main/scala/fr/acinq/eclair/tor/Socks5Connection.scala

@@ -0,0 +1,241 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.tor
+
+import java.net.{Inet4Address, Inet6Address, InetAddress, InetSocketAddress}
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated}
+import akka.io.Tcp
+import akka.util.ByteString
+import fr.acinq.bitcoin.toHexString
+import fr.acinq.eclair.randomBytes
+import fr.acinq.eclair.tor.Socks5Connection.{Credentials, Socks5Connect}
+import fr.acinq.eclair.wire._
+
+import scala.util.Success
+
+/**
+  * Simple socks 5 client. It should be given a new connection, and will
+  *
+  * Created by rorp
+  *
+  * @param connection      underlying TcpConnection
+  * @param credentials_opt optional username/password for authentication
+  */
+class Socks5Connection(connection: ActorRef, credentials_opt: Option[Credentials], command: Socks5Connect) extends Actor with ActorLogging {
+
+  import fr.acinq.eclair.tor.Socks5Connection._
+
+  context watch connection
+
+  val passwordAuth: Boolean = credentials_opt.isDefined
+
+  var isConnected: Boolean = false
+
+  connection ! Tcp.Register(self)
+  connection ! Tcp.ResumeReading
+  connection ! Tcp.Write(socks5Greeting(passwordAuth))
+
+  override def receive: Receive = greetings
+
+  def greetings: Receive = {
+    case Tcp.Received(data) =>
+        if (data(0) != 0x05) {
+          throw Socks5Error("Invalid SOCKS5 proxy response")
+        } else if ((!passwordAuth && data(1) != NoAuth) || (passwordAuth && data(1) != PasswordAuth)) {
+          throw Socks5Error("Unrecognized SOCKS5 auth method")
+        } else {
+          if (data(1) == PasswordAuth) {
+            context become authenticate
+            val credentials = credentials_opt.getOrElse(throw Socks5Error("credentials are not defined"))
+            connection ! Tcp.Write(socks5PasswordAuthenticationRequest(credentials.username, credentials.password))
+            connection ! Tcp.ResumeReading
+          } else {
+            context become connectionRequest
+            connection ! Tcp.Write(socks5ConnectionRequest(command.address))
+            connection ! Tcp.ResumeReading
+          }
+        }
+      }
+
+  def authenticate: Receive = {
+    case Tcp.Received(data) =>
+        if (data(0) != 0x01) {
+          throw Socks5Error("Invalid SOCKS5 proxy response")
+        } else if (data(1) != 0) {
+          throw Socks5Error("SOCKS5 authentication failed")
+        }
+        context become connectionRequest
+        connection ! Tcp.Write(socks5ConnectionRequest(command.address))
+        connection ! Tcp.ResumeReading
+      }
+
+  def connectionRequest: Receive = {
+    case Tcp.Received(data) =>
+        if (data(0) != 0x05) {
+          throw Socks5Error("Invalid SOCKS5 proxy response")
+        } else {
+          val status = data(1)
+          if (status != 0) {
+            throw Socks5Error(connectErrors.getOrElse(status, s"Unknown SOCKS5 error $status"))
+          }
+          val connectedAddress = data(3) match {
+            case 0x01 =>
+              val ip = Array(data(4), data(5), data(6), data(7))
+              val port = data(8).toInt << 8 | data(9)
+              new InetSocketAddress(InetAddress.getByAddress(ip), port)
+            case 0x03 =>
+              val len = data(4)
+              val start = 5
+              val end = start + len
+              val domain = data.slice(start, end).utf8String
+              val port = data(end).toInt << 8 | data(end + 1)
+              new InetSocketAddress(domain, port)
+            case 0x04 =>
+              val ip = Array.ofDim[Byte](16)
+              data.copyToArray(ip, 4, 4 + ip.length)
+              val port = data(4 + ip.length).toInt << 8 | data(4 + ip.length + 1)
+              new InetSocketAddress(InetAddress.getByAddress(ip), port)
+            case _ => throw Socks5Error(s"Unrecognized address type")
+          }
+          context become connected
+          context.parent ! Socks5Connected(connectedAddress)
+          isConnected = false
+        }
+  }
+
+  def connected: Receive = {
+    case Tcp.Register(handler, _, _) => context become registered(handler)
+  }
+
+  def registered(handler: ActorRef): Receive = {
+    case c: Tcp.Command => connection ! c
+    case e: Tcp.Event => handler ! e
+  }
+
+  override def unhandled(message: Any): Unit = message match {
+    case Terminated(actor) if actor == connection => context stop self
+    case _: Tcp.ConnectionClosed => context stop self
+    case _ => log.warning(s"unhandled message=$message")
+  }
+
+  override def postStop(): Unit = {
+    super.postStop()
+    connection ! Tcp.Close
+    if (!isConnected) {
+      context.parent ! command.failureMessage
+    }
+  }
+
+}
+
+object Socks5Connection {
+  def props(tcpConnection: ActorRef, credentials_opt: Option[Credentials], command: Socks5Connect): Props = Props(new Socks5Connection(tcpConnection, credentials_opt, command))
+
+  case class Socks5Connect(address: InetSocketAddress) extends Tcp.Command
+
+  case class Socks5Connected(address: InetSocketAddress) extends Tcp.Event
+
+  case class Socks5Error(message: String) extends RuntimeException(message)
+
+  case class Credentials(username: String, password: String) {
+    require(username.length < 256, "username is too long")
+    require(password.length < 256, "password is too long")
+  }
+
+  val NoAuth: Byte = 0x00
+  val PasswordAuth: Byte = 0x02
+
+  val connectErrors: Map[Byte, String] = Map[Byte, String](
+    (0x00, "Request granted"),
+    (0x01, "General failure"),
+    (0x02, "Connection not allowed by ruleset"),
+    (0x03, "Network unreachable"),
+    (0x04, "Host unreachable"),
+    (0x05, "Connection refused by destination host"),
+    (0x06, "TTL expired"),
+    (0x07, "Command not supported / protocol error"),
+    (0x08, "Address type not supported")
+  )
+
+  def socks5Greeting(passwordAuth: Boolean) = ByteString(
+    0x05, // SOCKS version
+    0x01, // number of authentication methods supported
+    if (passwordAuth) PasswordAuth else NoAuth) // auth method
+
+  def socks5PasswordAuthenticationRequest(username: String, password: String): ByteString =
+    ByteString(
+      0x01, // version of username/password authentication
+      username.length.toByte) ++
+      ByteString(username) ++
+      ByteString(password.length.toByte) ++
+      ByteString(password)
+
+  def socks5ConnectionRequest(address: InetSocketAddress): ByteString = {
+    ByteString(
+      0x05, // SOCKS version
+      0x01, // establish a TCP/IP stream connection
+      0x00) ++ // reserved
+      addressToByteString(address) ++
+      portToByteString(address.getPort)
+  }
+
+  def inetAddressToByteString(inet: InetAddress): ByteString = inet match {
+    case a: Inet4Address => ByteString(
+      0x01 // IPv4 address
+    ) ++ ByteString(a.getAddress)
+    case a: Inet6Address => ByteString(
+      0x04 // IPv6 address
+    ) ++ ByteString(a.getAddress)
+    case _ => throw Socks5Error("Unknown InetAddress")
+  }
+
+  def addressToByteString(address: InetSocketAddress): ByteString = Option(address.getAddress) match {
+    case None =>
+      // unresolved address, use SOCKS5 resolver
+      val host = address.getHostString
+      ByteString(
+        0x03, // Domain name
+        host.length.toByte) ++
+        ByteString(host)
+    case Some(inetAddress) =>
+      inetAddressToByteString(inetAddress)
+  }
+
+  def portToByteString(port: Int): ByteString = ByteString((port & 0x0000ff00) >> 8, port & 0x000000ff)
+}
+
+case class Socks5ProxyParams(address: InetSocketAddress, credentials_opt: Option[Credentials], randomizeCredentials: Boolean, useForIPv4: Boolean, useForIPv6: Boolean, useForTor: Boolean)
+
+object Socks5ProxyParams {
+
+  def proxyAddress(socketAddress: InetSocketAddress, proxyParams: Socks5ProxyParams): Option[InetSocketAddress] =
+    NodeAddress.fromParts(socketAddress.getHostString, socketAddress.getPort).toOption collect {
+      case _: IPv4 if proxyParams.useForIPv4 => proxyParams.address
+      case _: IPv6 if proxyParams.useForIPv6 => proxyParams.address
+      case _: Tor2 if proxyParams.useForTor => proxyParams.address
+      case _: Tor3 if proxyParams.useForTor => proxyParams.address
+    }
+
+  def proxyCredentials(proxyParams: Socks5ProxyParams): Option[Socks5Connection.Credentials] =
+    if (proxyParams.randomizeCredentials) {
+      // randomize credentials for every proxy connection to enable Tor stream isolation
+      Some(Socks5Connection.Credentials(toHexString(randomBytes(16)), toHexString(randomBytes(16))))
+    } else {
+      proxyParams.credentials_opt
+    }
+}

eclair-core/src/main/scala/fr/acinq/eclair/tor/TorProtocolHandler.scala

@@ -0,0 +1,311 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.tor
+
+import java.net.InetSocketAddress
+import java.nio.file.attribute.PosixFilePermissions
+import java.nio.file.{Files, Path, Paths}
+import java.util
+
+import akka.actor.{Actor, ActorLogging, ActorRef, Props, Stash}
+import akka.io.Tcp.Connected
+import akka.util.ByteString
+import fr.acinq.bitcoin.BinaryData
+import fr.acinq.eclair.tor.TorProtocolHandler.{Authentication, OnionServiceVersion}
+import fr.acinq.eclair.wire.{NodeAddress, Tor2, Tor3}
+import javax.crypto.Mac
+import javax.crypto.spec.SecretKeySpec
+
+import scala.concurrent.Promise
+import scala.util.Try
+
+case class TorException(private val msg: String) extends RuntimeException(s"Tor error: $msg")
+
+/**
+  * Created by rorp
+  *
+  * Specification: https://gitweb.torproject.org/torspec.git/tree/control-spec.txt
+  *
+  * @param onionServiceVersion v2 or v3
+  * @param authentication      Tor controller auth mechanism (password or safecookie)
+  * @param privateKeyPath      path to a file that contains a Tor private key
+  * @param virtualPort         port of our protected local server (typically 9735)
+  * @param targetPorts         target ports of the public hidden service
+  * @param onionAdded          a Promise to track creation of the endpoint
+  */
+class TorProtocolHandler(onionServiceVersion: OnionServiceVersion,
+                         authentication: Authentication,
+                         privateKeyPath: Path,
+                         virtualPort: Int,
+                         targetPorts: Seq[Int],
+                         onionAdded: Option[Promise[NodeAddress]]
+                        ) extends Actor with Stash with ActorLogging {
+
+  import TorProtocolHandler._
+
+  private var receiver: ActorRef = _
+
+  private var address: Option[NodeAddress] = None
+
+  override def receive: Receive = {
+    case Connected(_, _) =>
+      receiver = sender()
+      sendCommand("PROTOCOLINFO 1")
+      context become protocolInfo
+  }
+
+  def protocolInfo: Receive = {
+    case data: ByteString =>
+      val res = parseResponse(readResponse(data))
+      val methods: String = res.getOrElse("METHODS", throw TorException("auth methods not found"))
+      val torVersion = unquote(res.getOrElse("Tor", throw TorException("version not found")))
+      log.info(s"Tor version $torVersion")
+      if (!OnionServiceVersion.isCompatible(onionServiceVersion, torVersion)) {
+        throw TorException(s"version $torVersion does not support onion service $onionServiceVersion")
+      }
+      if (!Authentication.isCompatible(authentication, methods)) {
+        throw TorException(s"cannot use authentication '$authentication', supported methods are '$methods'")
+      }
+      authentication match {
+        case Password(password) =>
+          sendCommand(s"""AUTHENTICATE "$password"""")
+          context become authenticate
+        case SafeCookie(nonce) =>
+          val cookieFile = Paths.get(unquote(res.getOrElse("COOKIEFILE", throw TorException("cookie file not found"))))
+          sendCommand(s"AUTHCHALLENGE SAFECOOKIE $nonce")
+          context become cookieChallenge(cookieFile, nonce)
+      }
+  }
+
+  def cookieChallenge(cookieFile: Path, clientNonce: BinaryData): Receive = {
+    case data: ByteString =>
+      val res = parseResponse(readResponse(data))
+      val clientHash = computeClientHash(
+        res.getOrElse("SERVERHASH", throw TorException("server hash not found")),
+        res.getOrElse("SERVERNONCE", throw TorException("server nonce not found")),
+        clientNonce,
+        cookieFile
+      )
+      sendCommand(s"AUTHENTICATE $clientHash")
+      context become authenticate
+  }
+
+  def authenticate: Receive = {
+    case data: ByteString =>
+      readResponse(data)
+      sendCommand(s"ADD_ONION $computeKey $computePort")
+      context become addOnion
+  }
+
+  def addOnion: Receive = {
+    case data: ByteString =>
+      val res = readResponse(data)
+      if (ok(res)) {
+        val serviceId = processOnionResponse(parseResponse(res))
+        address = Some(onionServiceVersion match {
+          case V2 => Tor2(serviceId, virtualPort)
+          case V3 => Tor3(serviceId, virtualPort)
+        })
+        onionAdded.foreach(_.success(address.get))
+        log.debug(s"Onion address: ${address.get}")
+      }
+  }
+
+
+  override def aroundReceive(receive: Receive, msg: Any): Unit = try {
+    super.aroundReceive(receive, msg)
+  } catch {
+    case t: Throwable => onionAdded.map(_.tryFailure(t))
+  }
+
+  override def unhandled(message: Any): Unit = message match {
+    case GetOnionAddress =>
+      sender() ! address
+  }
+
+  private def processOnionResponse(res: Map[String, String]): String = {
+    val serviceId = res.getOrElse("ServiceID", throw TorException("service ID not found"))
+    val privateKey = res.get("PrivateKey")
+    privateKey.foreach { pk =>
+      writeString(privateKeyPath, pk)
+      setPermissions(privateKeyPath, "rw-------")
+    }
+    serviceId
+  }
+
+  private def computeKey: String = {
+    if (privateKeyPath.toFile.exists()) {
+      readString(privateKeyPath)
+    } else {
+      onionServiceVersion match {
+        case V2 => "NEW:RSA1024"
+        case V3 => "NEW:ED25519-V3"
+      }
+    }
+  }
+
+  private def computePort: String = {
+    if (targetPorts.isEmpty) {
+      s"Port=$virtualPort,$virtualPort"
+    } else {
+      targetPorts.map(p => s"Port=$virtualPort,$p").mkString(" ")
+    }
+  }
+
+  private def computeClientHash(serverHash: BinaryData, serverNonce: BinaryData, clientNonce: BinaryData, cookieFile: Path): BinaryData = {
+    if (serverHash.length != 32)
+      throw TorException("invalid server hash length")
+    if (serverNonce.length != 32)
+      throw TorException("invalid server nonce length")
+
+    val cookie = Files.readAllBytes(cookieFile)
+
+    val message = cookie ++ clientNonce ++ serverNonce
+
+    val computedServerHash = hmacSHA256(ServerKey, message)
+    if (computedServerHash != serverHash) {
+      throw TorException("unexpected server hash")
+    }
+
+    hmacSHA256(ClientKey, message)
+  }
+
+  private def sendCommand(cmd: String): Unit = {
+    receiver ! ByteString(s"$cmd\r\n")
+  }
+}
+
+object TorProtocolHandler {
+  def props(version: OnionServiceVersion,
+            authentication: Authentication,
+            privateKeyPath: Path,
+            virtualPort: Int,
+            targetPorts: Seq[Int] = Seq(),
+            onionAdded: Option[Promise[NodeAddress]] = None
+           ): Props =
+    Props(new TorProtocolHandler(version, authentication, privateKeyPath, virtualPort, targetPorts, onionAdded))
+
+  // those are defined in the spec
+  private val ServerKey: Array[Byte] = "Tor safe cookie authentication server-to-controller hash".getBytes()
+  private val ClientKey: Array[Byte] = "Tor safe cookie authentication controller-to-server hash".getBytes()
+
+  // @formatter:off
+  sealed trait OnionServiceVersion
+  case object V2 extends OnionServiceVersion
+  case object V3 extends OnionServiceVersion
+  // @formatter:on
+
+  object OnionServiceVersion {
+    def apply(s: String): OnionServiceVersion = s match {
+      case "v2" | "V2" => V2
+      case "v3" | "V3" => V3
+      case _ => throw TorException(s"unknown protocol version `$s`")
+    }
+
+    def isCompatible(onionServiceVersion: OnionServiceVersion, torVersion: String): Boolean =
+      onionServiceVersion match {
+        case V2 => true
+        case V3 => torVersion
+            .split("\\.")
+            .map(_.split('-').head) // remove non-numeric symbols at the end of the last number (rc, beta, alpha, etc.)
+            .map(d => Try(d.toInt).getOrElse(0))
+            .zipAll(List(0, 3, 3, 6), 0, 0) // min version for v3 is 0.3.3.6
+            .foldLeft(Option.empty[Boolean]) { // compare subversion by subversion starting from the left
+              case (Some(res), _) => Some(res) // we stop the comparison as soon as there is a difference
+              case (None, (v, vref)) => if (v > vref) Some(true) else if (v < vref) Some(false) else None
+            }
+            .getOrElse(true) // if version == 0.3.3.6 then result will be None
+
+      }
+  }
+
+  // @formatter:off
+  sealed trait Authentication
+  case class Password(password: String)                                              extends Authentication { override def toString = "password" }
+  case class SafeCookie(nonce: BinaryData = fr.acinq.eclair.randomBytes(32)) extends Authentication { override def toString = "safecookie" }
+  // @formatter:on
+
+  object Authentication {
+    def isCompatible(authentication: Authentication, methods: String): Boolean =
+      authentication match {
+        case _: Password => methods.contains("HASHEDPASSWORD")
+        case _: SafeCookie => methods.contains("SAFECOOKIE")
+      }
+  }
+
+  case object GetOnionAddress
+
+  def readString(path: Path): String = Files.readAllLines(path).get(0)
+
+  def writeString(path: Path, string: String): Unit = Files.write(path, util.Arrays.asList(string))
+
+  def setPermissions(path: Path, permissionString: String): Unit =
+    try {
+      Files.setPosixFilePermissions(path, PosixFilePermissions.fromString(permissionString))
+    } catch {
+      case _: UnsupportedOperationException => () // we are on windows
+    }
+
+  def unquote(s: String): String = s
+    .stripSuffix("\"")
+    .stripPrefix("\"")
+    .replace("""\\""", """\""")
+    .replace("""\"""", "\"")
+
+  private val r1 = """(\d+)\-(.*)""".r
+  private val r2 = """(\d+) (.*)""".r
+
+  def readResponse(bstr: ByteString): Seq[(Int, String)] = {
+    val lines = bstr.utf8String.split('\n')
+      .map(_.stripSuffix("\r"))
+      .filterNot(_.isEmpty)
+      .map {
+        case r1(c, msg) => (c.toInt, msg)
+        case r2(c, msg) => (c.toInt, msg)
+        case x@_ => throw TorException(s"unknown response line format: `$x`")
+      }
+    if (!ok(lines)) {
+      throw TorException(s"server returned error: ${status(lines)} ${reason(lines)}")
+    }
+    lines
+  }
+
+  def ok(res: Seq[(Int, String)]): Boolean = status(res) == 250
+
+  def status(res: Seq[(Int, String)]): Int = res.lastOption.map(_._1).getOrElse(-1)
+
+  def reason(res: Seq[(Int, String)]): String = res.lastOption.map(_._2).getOrElse("Unknown error")
+
+  private val r = """([^=]+)=(.+)""".r
+
+  def parseResponse(lines: Seq[(Int, String)]): Map[String, String] = {
+    lines.flatMap {
+      case (_, message) =>
+        message.split(" ")
+          .collect {
+            case r(k, v) => (k, v)
+          }
+    }.toMap
+  }
+
+  def hmacSHA256(key: Array[Byte], message: Array[Byte]): BinaryData = {
+    val mac = Mac.getInstance("HmacSHA256")
+    val secretKey = new SecretKeySpec(key, "HmacSHA256")
+    mac.init(secretKey)
+    mac.doFinal(message)
+  }
+}

eclair-core/src/main/scala/fr/acinq/eclair/wire/LightningMessageCodecs.scala

@@ -18,12 +18,14 @@ package fr.acinq.eclair.wire
 
 import java.math.BigInteger
 import java.net.{Inet4Address, Inet6Address, InetAddress}
+
 import com.google.common.cache.{CacheBuilder, CacheLoader}
 import fr.acinq.bitcoin.Crypto.{Point, PrivateKey, PublicKey, Scalar}
 import fr.acinq.bitcoin.{BinaryData, Crypto}
 import fr.acinq.eclair.crypto.{Generators, Sphinx}
 import fr.acinq.eclair.wire.FixedSizeStrictCodec.bytesStrict
 import fr.acinq.eclair.{ShortChannelId, UInt64, wire}
+import org.apache.commons.codec.binary.Base32
 import scodec.bits.{BitVector, ByteVector}
 import scodec.codecs._
 import scodec.{Attempt, Codec, DecodeResult, Err, SizeBound}
@@ -58,13 +60,14 @@ object LightningMessageCodecs {
 
   def ipv6address: Codec[Inet6Address] = bytes(16).exmap(b => attemptFromTry(Inet6Address.getByAddress(null, b.toArray, null)), a => attemptFromTry(ByteVector(a.getAddress)))
 
+  def base32(size: Int): Codec[String] = bytes(size).xmap(b => new Base32().encodeAsString(b.toArray).toLowerCase, a => ByteVector(new Base32().decode(a.toUpperCase())))
+
   def nodeaddress: Codec[NodeAddress] =
     discriminated[NodeAddress].by(uint8)
-      .typecase(0, provide(Padding))
+      .typecase(1, (ipv4address :: uint16).as[IPv4])
-      .typecase(1, (ipv4address ~ uint16).xmap[IPv4](x => new IPv4(x._1, x._2), x => (x.ipv4, x.port)))
+      .typecase(2, (ipv6address :: uint16).as[IPv6])
-      .typecase(2, (ipv6address ~ uint16).xmap[IPv6](x => new IPv6(x._1, x._2), x => (x.ipv6, x.port)))
+      .typecase(3, (base32(10) :: uint16).as[Tor2])
-      .typecase(3, (binarydata(10) ~ uint16).xmap[Tor2](x => new Tor2(x._1, x._2), x => (x.tor2, x.port)))
+      .typecase(4, (base32(35) :: uint16).as[Tor3])
-      .typecase(4, (binarydata(35) ~ uint16).xmap[Tor3](x => new Tor3(x._1, x._2), x => (x.tor3, x.port)))
 
   // this one is a bit different from most other codecs: the first 'len' element is *not* the number of items
   // in the list but rather the  number of bytes of the encoded list. The rationale is once we've read this

eclair-core/src/main/scala/fr/acinq/eclair/wire/LightningMessageTypes.scala

@@ -16,12 +16,14 @@
 
 package fr.acinq.eclair.wire
 
-import java.net.{Inet4Address, Inet6Address, InetSocketAddress}
+import java.net.{Inet4Address, Inet6Address, InetAddress, InetSocketAddress}
 
+import com.google.common.net.HostAndPort
 import fr.acinq.bitcoin.BinaryData
 import fr.acinq.bitcoin.Crypto.{Point, PublicKey, Scalar}
 import fr.acinq.eclair.{ShortChannelId, UInt64}
-import scodec.bits.BitVector
+
+import scala.util.{Success, Try}
 
 /**
   * Created by PM on 15/11/2016.
@@ -161,33 +163,46 @@ case class Color(r: Byte, g: Byte, b: Byte) {
 }
 
 // @formatter:off
-sealed trait NodeAddress
+sealed trait NodeAddress { def socketAddress: InetSocketAddress }
-case object NodeAddress {
+sealed trait OnionAddress extends NodeAddress
-  def apply(inetSocketAddress: InetSocketAddress): NodeAddress = inetSocketAddress.getAddress match {
+object NodeAddress {
-    case a: Inet4Address => IPv4(a, inetSocketAddress.getPort)
+  /**
-    case a: Inet6Address => IPv6(a, inetSocketAddress.getPort)
+    * Creates a NodeAddress from a host and port.
-    case _ => throw new RuntimeException(s"Invalid socket address $inetSocketAddress")
+    *
+    * Note that non-onion hosts will be resolved.
+    *
+    * We don't attempt to resolve onion addresses (it will be done by the tor proxy), so we just recognize them based on
+    * the .onion TLD and rely on their length to separate v2/v3.
+    *
+    * @param host
+    * @param port
+    * @return
+    */
+  def fromParts(host: String, port: Int): Try[NodeAddress] = Try {
+    host match {
+      case _ if host.endsWith(".onion") && host.length == 22 => Tor2(host.dropRight(6), port)
+      case _ if host.endsWith(".onion") && host.length == 62 => Tor3(host.dropRight(6), port)
+      case _  => InetAddress.getByName(host) match {
+        case a: Inet4Address => IPv4(a, port)
+        case a: Inet6Address => IPv6(a, port)
       }
     }
-case object Padding extends NodeAddress
+  }
-case class IPv4(ipv4: Inet4Address, port: Int) extends NodeAddress
+}
-case class IPv6(ipv6: Inet6Address, port: Int) extends NodeAddress
+case class IPv4(ipv4: Inet4Address, port: Int) extends NodeAddress { override def socketAddress = new InetSocketAddress(ipv4, port) }
-case class Tor2(tor2: BinaryData, port: Int) extends NodeAddress { require(tor2.size == 10) }
+case class IPv6(ipv6: Inet6Address, port: Int) extends NodeAddress { override def socketAddress = new InetSocketAddress(ipv6, port) }
-case class Tor3(tor3: BinaryData, port: Int) extends NodeAddress { require(tor3.size == 35) }
+case class Tor2(tor2: String, port: Int) extends OnionAddress { override def socketAddress = InetSocketAddress.createUnresolved(tor2 + ".onion", port) }
+case class Tor3(tor3: String, port: Int) extends OnionAddress { override def socketAddress = InetSocketAddress.createUnresolved(tor3 + ".onion", port) }
 // @formatter:on
 
+
 case class NodeAnnouncement(signature: BinaryData,
                             features: BinaryData,
                             timestamp: Long,
                             nodeId: PublicKey,
                             rgbColor: Color,
                             alias: String,
-                            addresses: List[NodeAddress]) extends RoutingMessage with HasTimestamp {
+                            addresses: List[NodeAddress]) extends RoutingMessage with HasTimestamp
-  def socketAddresses: List[InetSocketAddress] = addresses.collect {
-    case IPv4(a, port) => new InetSocketAddress(a, port)
-    case IPv6(a, port) => new InetSocketAddress(a, port)
-  }
-}
 
 case class ChannelUpdate(signature: BinaryData,
                          chainHash: BinaryData,

eclair-core/src/test/resources/api/getinfo

@@ -1,5 +1,6 @@
 {
   "result" : {
+    "publicAddresses" : [ "localhost:9731" ],
     "alias" : "alice",
     "port" : 9735,
     "chainHash" : "06226e46111a0b59caaf126043eb5bbf28c34f3a5e332a1fc7b2b73cf188910f",

eclair-core/src/test/scala/fr/acinq/eclair/StartupSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair
 
 import java.io.{File, IOException}
@@ -34,7 +50,7 @@ class StartupSpec extends FunSuite {
     val keyManager = new LocalKeyManager(seed = randomKey.toBin, chainHash = Block.TestnetGenesisBlock.hash)
 
     // try to create a NodeParams instance with a conf that contains an illegal alias
-    val nodeParamsAttempt = Try(NodeParams.makeNodeParams(tempConfParentDir, conf, keyManager))
+    val nodeParamsAttempt = Try(NodeParams.makeNodeParams(tempConfParentDir, conf, keyManager, None))
     assert(nodeParamsAttempt.isFailure && nodeParamsAttempt.failed.get.getMessage.contains("alias, too long"))
 
     // destroy conf files after the test

eclair-core/src/test/scala/fr/acinq/eclair/TestConstants.scala

@@ -16,16 +16,16 @@
 
 package fr.acinq.eclair
 
-import java.net.InetSocketAddress
 import java.sql.DriverManager
 
+import com.google.common.net.HostAndPort
 import fr.acinq.bitcoin.Crypto.PrivateKey
 import fr.acinq.bitcoin.{BinaryData, Block, Script}
 import fr.acinq.eclair.NodeParams.BITCOIND
 import fr.acinq.eclair.crypto.LocalKeyManager
 import fr.acinq.eclair.db.sqlite._
 import fr.acinq.eclair.io.Peer
-import fr.acinq.eclair.wire.Color
+import fr.acinq.eclair.wire.{Color, NodeAddress}
 
 import scala.concurrent.duration._
 
@@ -48,7 +48,7 @@ object TestConstants {
       keyManager = keyManager,
       alias = "alice",
       color = Color(1, 2, 3),
-      publicAddresses = new InetSocketAddress("localhost", 9731) :: Nil,
+      publicAddresses = NodeAddress.fromParts("localhost", 9731).get :: Nil,
       globalFeatures = "",
       localFeatures = "00",
       overrideFeatures = Map.empty,
@@ -86,7 +86,9 @@ object TestConstants {
       paymentRequestExpiry = 1 hour,
       maxPendingPaymentRequests = 10000000,
       maxPaymentFee = 0.03,
-      minFundingSatoshis = 1000L)
+      minFundingSatoshis = 1000L,
+      randomizeRouteSelection = true,
+      socksProxy_opt = None)
 
     def channelParams = Peer.makeChannelParams(
       nodeParams = nodeParams,
@@ -107,7 +109,7 @@ object TestConstants {
       keyManager = keyManager,
       alias = "bob",
       color = Color(4, 5, 6),
-      publicAddresses = new InetSocketAddress("localhost", 9732) :: Nil,
+      publicAddresses = NodeAddress.fromParts("localhost", 9732).get :: Nil,
       globalFeatures = "",
       localFeatures = "00", // no announcement
       overrideFeatures = Map.empty,
@@ -145,7 +147,9 @@ object TestConstants {
       paymentRequestExpiry = 1 hour,
       maxPendingPaymentRequests = 10000000,
       maxPaymentFee = 0.03,
-      minFundingSatoshis = 1000L)
+      minFundingSatoshis = 1000L,
+      randomizeRouteSelection = true,
+      socksProxy_opt = None)
 
     def channelParams = Peer.makeChannelParams(
       nodeParams = nodeParams,

eclair-core/src/test/scala/fr/acinq/eclair/TestUtils.scala

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair
+
+import java.io.File
+
+object TestUtils {
+
+  /**
+    * Get the module's target directory (works from command line and within intellij)
+    */
+  val BUILD_DIRECTORY = sys
+    .props
+    .get("buildDirectory") // this is defined if we run from maven
+    .getOrElse(new File(sys.props("user.dir"), "target").getAbsolutePath) // otherwise we probably are in intellij, so we build it manually assuming that user.dir == path to the module
+
+}

eclair-core/src/test/scala/fr/acinq/eclair/api/JsonRpcServiceSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.api
 
 
@@ -143,7 +159,7 @@ class JsonRpcServiceSpec extends FunSuite with ScalatestRouteTest {
         case GetPeerInfo => sender() ! PeerInfo(
           nodeId = Alice.nodeParams.nodeId,
           state = "CONNECTED",
-          address = Some(Alice.nodeParams.publicAddresses.head),
+          address = Some(Alice.nodeParams.publicAddresses.head.socketAddress),
           channels = 1)
       }
     }))
@@ -162,7 +178,7 @@ class JsonRpcServiceSpec extends FunSuite with ScalatestRouteTest {
     val mockService = new MockService(defaultMockKit.copy(
       switchboard = system.actorOf(Props(new {} with MockActor {
         override def receive = {
-          case 'peers => sender() ! Map(Alice.nodeParams.nodeId -> mockAlicePeer, Bob.nodeParams.nodeId -> mockBobPeer)
+          case 'peers => sender() ! List(mockAlicePeer, mockBobPeer)
         }
       }))
     ))
@@ -195,7 +211,8 @@ class JsonRpcServiceSpec extends FunSuite with ScalatestRouteTest {
         alias = Alice.nodeParams.alias,
         port = 9735,
         chainHash = Alice.nodeParams.chainHash,
-        blockHeight = 123456
+        blockHeight = 123456,
+        publicAddresses = Alice.nodeParams.publicAddresses
       ))
     }
     import mockService.formats

eclair-core/src/test/scala/fr/acinq/eclair/api/JsonSerializersSpec.scala

@@ -18,10 +18,11 @@ package fr.acinq.eclair.api
 
 import java.net.{InetAddress, InetSocketAddress}
 
+import com.google.common.net.HostAndPort
 import fr.acinq.bitcoin.{BinaryData, OutPoint}
 import fr.acinq.eclair.payment.PaymentRequest
 import fr.acinq.eclair.transactions.{IN, OUT}
-import fr.acinq.eclair.wire.NodeAddress
+import fr.acinq.eclair.wire.{NodeAddress, Tor2, Tor3}
 import org.json4s.jackson.Serialization
 import org.scalatest.{FunSuite, Matchers}
 
@@ -50,11 +51,15 @@ class JsonSerializersSpec extends FunSuite with Matchers {
   }
 
   test("NodeAddress serialization") {
-    val ipv4 = NodeAddress(new InetSocketAddress(InetAddress.getByAddress(Array(10, 0, 0, 1)), 8888))
+    val ipv4 = NodeAddress.fromParts("10.0.0.1", 8888).get
-    val ipv6LocalHost = NodeAddress(new InetSocketAddress(InetAddress.getByAddress(Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)), 9735))
+    val ipv6LocalHost = NodeAddress.fromParts(InetAddress.getByAddress(Array(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)).getHostAddress, 9735).get
+    val tor2 = Tor2("aaaqeayeaudaocaj", 7777)
+    val tor3 = Tor3("aaaqeayeaudaocajbifqydiob4ibceqtcqkrmfyydenbwha5dypsaijc", 9999)
 
     Serialization.write(ipv4)(org.json4s.DefaultFormats + new NodeAddressSerializer) shouldBe s""""10.0.0.1:8888""""
     Serialization.write(ipv6LocalHost)(org.json4s.DefaultFormats + new NodeAddressSerializer) shouldBe s""""[0:0:0:0:0:0:0:1]:9735""""
+    Serialization.write(tor2)(org.json4s.DefaultFormats + new NodeAddressSerializer) shouldBe s""""aaaqeayeaudaocaj.onion:7777""""
+    Serialization.write(tor3)(org.json4s.DefaultFormats + new NodeAddressSerializer) shouldBe s""""aaaqeayeaudaocajbifqydiob4ibceqtcqkrmfyydenbwha5dypsaijc.onion:9999""""
   }
 
   test("Direction serialization") {

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/bitcoind/BitcoindService.scala

@@ -25,6 +25,7 @@ import akka.pattern.pipe
 import akka.testkit.{TestKitBase, TestProbe}
 import com.softwaremill.sttp.okhttp.OkHttpFutureBackend
 import com.softwaremill.sttp.okhttp.OkHttpFutureBackend
+import fr.acinq.eclair.TestUtils
 import fr.acinq.eclair.blockchain.bitcoind.rpc.{BasicBitcoinJsonRPCClient, BitcoinJsonRPCClient}
 import fr.acinq.eclair.integration.IntegrationSpec
 import grizzled.slf4j.Logging
@@ -41,10 +42,10 @@ trait BitcoindService extends Logging {
 
   import scala.sys.process._
 
-  val INTEGRATION_TMP_DIR = s"${System.getProperty("buildDirectory")}/integration-${UUID.randomUUID().toString}"
+  val INTEGRATION_TMP_DIR = new File(TestUtils.BUILD_DIRECTORY, s"integration-${UUID.randomUUID()}")
   logger.info(s"using tmp dir: $INTEGRATION_TMP_DIR")
 
-  val PATH_BITCOIND = new File(System.getProperty("buildDirectory"), "bitcoin-0.16.3/bin/bitcoind")
+  val PATH_BITCOIND = new File(TestUtils.BUILD_DIRECTORY, "bitcoin-0.16.3/bin/bitcoind")
   val PATH_BITCOIND_DATADIR = new File(INTEGRATION_TMP_DIR, "datadir-bitcoin")
 
   var bitcoind: Process = null

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWalletBasicSpec.scala

@@ -57,9 +57,9 @@ class ElectrumWalletBasicSpec extends FunSuite with Logging {
   def addFunds(data: Data, key: ExtendedPrivateKey, amount: Satoshi): Data = {
     val tx = Transaction(version = 1, txIn = Nil, txOut = TxOut(amount, ElectrumWallet.computePublicKeyScript(key.publicKey)) :: Nil, lockTime = 0)
     val scriptHash = ElectrumWallet.computeScriptHashFromPublicKey(key.publicKey)
-    val scriptHashHistory = data.history.getOrElse(scriptHash, Seq.empty[ElectrumClient.TransactionHistoryItem])
+    val scriptHashHistory = data.history.getOrElse(scriptHash, List.empty[ElectrumClient.TransactionHistoryItem])
     data.copy(
-      history = data.history.updated(scriptHash, scriptHashHistory :+ ElectrumClient.TransactionHistoryItem(100, tx.txid)),
+      history = data.history.updated(scriptHash, ElectrumClient.TransactionHistoryItem(100, tx.txid) :: scriptHashHistory),
       transactions = data.transactions + (tx.txid -> tx)
     )
   }
@@ -67,9 +67,9 @@ class ElectrumWalletBasicSpec extends FunSuite with Logging {
   def addFunds(data: Data, keyamount: (ExtendedPrivateKey, Satoshi)): Data = {
     val tx = Transaction(version = 1, txIn = Nil, txOut = TxOut(keyamount._2, ElectrumWallet.computePublicKeyScript(keyamount._1.publicKey)) :: Nil, lockTime = 0)
     val scriptHash = ElectrumWallet.computeScriptHashFromPublicKey(keyamount._1.publicKey)
-    val scriptHashHistory = data.history.getOrElse(scriptHash, Seq.empty[ElectrumClient.TransactionHistoryItem])
+    val scriptHashHistory = data.history.getOrElse(scriptHash, List.empty[ElectrumClient.TransactionHistoryItem])
     data.copy(
-      history = data.history.updated(scriptHash, scriptHashHistory :+ ElectrumClient.TransactionHistoryItem(100, tx.txid)),
+      history = data.history.updated(scriptHash, ElectrumClient.TransactionHistoryItem(100, tx.txid) :: scriptHashHistory),
       transactions = data.transactions + (tx.txid -> tx)
     )
   }

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWalletSimulatedClientSpec.scala

@@ -185,7 +185,7 @@ class ElectrumWalletSimulatedClientSpec extends TestKit(ActorSystem("test")) wit
 
     client.expectMsg(GetTransaction(tx.txid))
     wallet ! GetTransactionResponse(tx)
-    val TransactionReceived(_, _, Satoshi(100000), _, _) = listener.expectMsgType[TransactionReceived]
+    val TransactionReceived(_, _, Satoshi(100000), _, _, _) = listener.expectMsgType[TransactionReceived]
     // we think we have some unconfirmed funds
     val WalletReady(Satoshi(100000), _, _, _) = listener.expectMsgType[WalletReady]
 

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/ElectrumWalletSpec.scala

@@ -196,7 +196,7 @@ class ElectrumWalletSpec extends TestKit(ActorSystem("test")) with FunSuiteLike
       unconfirmed1 - unconfirmed == Satoshi(100000000L)
     }, max = 30 seconds, interval = 1 second)
 
-    val TransactionReceived(tx, 0, received, sent, _) = listener.receiveOne(5 seconds)
+    val TransactionReceived(tx, 0, received, sent, _, _) = listener.receiveOne(5 seconds)
     assert(tx.txid === BinaryData(txid))
     assert(received === Satoshi(100000000))
 
@@ -211,7 +211,10 @@ class ElectrumWalletSpec extends TestKit(ActorSystem("test")) with FunSuiteLike
 
     awaitCond({
       val msg = listener.receiveOne(5 seconds)
-      msg == TransactionConfidenceChanged(BinaryData(txid), 1)
+      msg match {
+        case TransactionConfidenceChanged(BinaryData(txid), 1, _) => true
+        case _ => false
+      }
     }, max = 30 seconds, interval = 1 second)
   }
 

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/electrum/db/sqlite/SqliteWalletDbSpec.scala

@@ -1,12 +1,31 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.blockchain.electrum.db.sqlite
 
 import java.sql.DriverManager
-import java.util.Random
 
-import fr.acinq.bitcoin.{BinaryData, Block, BlockHeader, Transaction}
+import fr.acinq.bitcoin.{BinaryData, Block, BlockHeader, OutPoint, Satoshi, Transaction, TxIn, TxOut}
+import fr.acinq.eclair.blockchain.electrum.ElectrumClient
 import fr.acinq.eclair.blockchain.electrum.ElectrumClient.GetMerkleResponse
+import fr.acinq.eclair.blockchain.electrum.ElectrumWallet.PersistentData
 import org.scalatest.FunSuite
 
+import scala.util.Random
+
 class SqliteWalletDbSpec extends FunSuite {
   val random = new Random()
 
@@ -41,14 +60,50 @@ class SqliteWalletDbSpec extends FunSuite {
     })
   }
 
-  test("add/get/list transactions") {
+  test("serialize persistent data") {
     val db = new SqliteWalletDb(inmem)
-    val tx = Transaction.read("0100000001b021a77dcaad3a2da6f1611d2403e1298a902af8567c25d6e65073f6b52ef12d000000006a473044022056156e9f0ad7506621bc1eb963f5133d06d7259e27b13fcb2803f39c7787a81c022056325330585e4be39bcf63af8090a2deff265bc29a3fb9b4bf7a31426d9798150121022dfb538041f111bb16402aa83bd6a3771fa8aa0e5e9b0b549674857fafaf4fe0ffffffff0210270000000000001976a91415c23e7f4f919e9ff554ec585cb2a67df952397488ac3c9d1000000000001976a9148982824e057ccc8d4591982df71aa9220236a63888ac00000000")
-    val proof = GetMerkleResponse(tx.hash, List(BinaryData("01" * 32), BinaryData("02" * 32)), 100000, 15)
-    db.addTransaction(tx, proof)
 
-    val Some((tx1, proof1)) = db.getTransaction(tx.hash)
+    def randomBytes(size: Int): BinaryData = {
-    assert(tx1 == tx)
+      val buffer = new Array[Byte](size)
-    assert(proof1 == proof)
+      random.nextBytes(buffer)
+      buffer
+    }
+
+    def randomTransaction = Transaction(version = 2,
+      txIn = TxIn(OutPoint(randomBytes(32), random.nextInt(100)), signatureScript = Nil, sequence = TxIn.SEQUENCE_FINAL) :: Nil,
+      txOut = TxOut(Satoshi(random.nextInt(10000000)), randomBytes(20)) :: Nil,
+      0L
+    )
+
+    def randomHistoryItem = ElectrumClient.TransactionHistoryItem(random.nextInt(1000000), randomBytes(32))
+
+    def randomHistoryItems = (0 to random.nextInt(100)).map(_ => randomHistoryItem).toList
+
+    def randomProof = GetMerkleResponse(randomBytes(32), ((0 until 10).map(_ => randomBytes(32))).toList, random.nextInt(100000), 0)
+
+    def randomPersistentData = {
+      val transactions = for (i <- 0 until random.nextInt(100)) yield randomTransaction
+
+      PersistentData(
+        accountKeysCount = 10,
+        changeKeysCount = 10,
+        status = (for (i <- 0 until random.nextInt(100)) yield randomBytes(32) -> random.nextInt(100000).toHexString).toMap,
+        transactions = transactions.map(tx => tx.hash -> tx).toMap,
+        heights = transactions.map(tx => tx.hash -> random.nextInt(500000).toLong).toMap,
+        history = (for (i <- 0 until random.nextInt(100)) yield randomBytes(32) -> randomHistoryItems).toMap,
+        proofs = (for (i <- 0 until random.nextInt(100)) yield randomBytes(32) -> randomProof).toMap,
+        pendingTransactions = transactions.toList,
+        locks = (for (i <- 0 until random.nextInt(10)) yield randomTransaction).toSet
+      )
+    }
+
+    assert(db.readPersistentData() == None)
+
+    for (i <- 0 until 50) {
+      val data = randomPersistentData
+      db.persist(data)
+      val Some(check) = db.readPersistentData()
+      assert(check === data)
+    }
   }
 }

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/fee/BitcoinCoreFeeProviderSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.blockchain.fee
 
 import akka.actor.ActorSystem

eclair-core/src/test/scala/fr/acinq/eclair/blockchain/fee/SmoothFeeProviderSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.blockchain.fee
 
 import org.scalatest.FunSuite

eclair-core/src/test/scala/fr/acinq/eclair/crypto/TransportHandlerSpec.scala

@@ -16,6 +16,7 @@
 
 package fr.acinq.eclair.crypto
 
+import java.net.InetSocketAddress
 import java.nio.charset.Charset
 
 import akka.actor.{Actor, ActorLogging, ActorRef, ActorSystem, OneForOneStrategy, Props, Stash, SupervisorStrategy, Terminated}

eclair-core/src/test/scala/fr/acinq/eclair/db/SqliteAuditDbSpec.scala

@@ -19,10 +19,10 @@ package fr.acinq.eclair.db
 import java.sql.DriverManager
 
 import fr.acinq.bitcoin.{MilliSatoshi, Satoshi, Transaction}
-import fr.acinq.eclair.channel.NetworkFeePaid
+import fr.acinq.eclair.channel.{AvailableBalanceChanged, NetworkFeePaid}
 import fr.acinq.eclair.db.sqlite.SqliteAuditDb
 import fr.acinq.eclair.payment.{PaymentReceived, PaymentRelayed, PaymentSent}
-import fr.acinq.eclair.{randomBytes, randomKey}
+import fr.acinq.eclair.{ShortChannelId, randomBytes, randomKey}
 import org.scalatest.FunSuite
 
 import scala.compat.Platform
@@ -48,6 +48,8 @@ class SqliteAuditDbSpec extends FunSuite {
     val e4 = NetworkFeePaid(null, randomKey.publicKey, randomBytes(32), Transaction(0, Seq.empty, Seq.empty, 0), Satoshi(42), "mutual")
     val e5 = PaymentSent(MilliSatoshi(42000), MilliSatoshi(1000), randomBytes(32), randomBytes(32), randomBytes(32), timestamp = 0)
     val e6 = PaymentSent(MilliSatoshi(42000), MilliSatoshi(1000), randomBytes(32), randomBytes(32), randomBytes(32), timestamp = Platform.currentTime * 2)
+    val e7 = AvailableBalanceChanged(null, randomBytes(32), ShortChannelId(500000, 42, 1), 456123000, ChannelStateSpec.commitments)
+    val e8 = ChannelLifecycleEvent(randomBytes(32), randomKey.publicKey, 456123000, true, false, "mutual")
 
     db.add(e1)
     db.add(e2)
@@ -55,6 +57,8 @@ class SqliteAuditDbSpec extends FunSuite {
     db.add(e4)
     db.add(e5)
     db.add(e6)
+    db.add(e7)
+    db.add(e8)
 
     assert(db.listSent(from = 0L, to = Long.MaxValue).toSet === Set(e1, e5, e6))
     assert(db.listSent(from = 100000L, to = Platform.currentTime + 1).toList === List(e1))

eclair-core/src/test/scala/fr/acinq/eclair/db/SqliteNetworkDbSpec.scala

@@ -19,10 +19,11 @@ package fr.acinq.eclair.db
 import java.net.{InetAddress, InetSocketAddress}
 import java.sql.DriverManager
 
+import com.google.common.net.HostAndPort
 import fr.acinq.bitcoin.{Block, Crypto, Satoshi}
 import fr.acinq.eclair.db.sqlite.SqliteNetworkDb
 import fr.acinq.eclair.router.Announcements
-import fr.acinq.eclair.wire.Color
+import fr.acinq.eclair.wire.{Color, NodeAddress, Tor2}
 import fr.acinq.eclair.{ShortChannelId, randomKey}
 import org.scalatest.FunSuite
 import org.sqlite.SQLiteException
@@ -42,9 +43,10 @@ class SqliteNetworkDbSpec extends FunSuite {
     val sqlite = inmem
     val db = new SqliteNetworkDb(sqlite)
 
-    val node_1 = Announcements.makeNodeAnnouncement(randomKey, "node-alice", Color(100.toByte, 200.toByte, 300.toByte), new InetSocketAddress(InetAddress.getByAddress(Array[Byte](192.toByte, 168.toByte, 1.toByte, 42.toByte)), 42000) :: Nil)
+    val node_1 = Announcements.makeNodeAnnouncement(randomKey, "node-alice", Color(100.toByte, 200.toByte, 300.toByte), NodeAddress.fromParts("192.168.1.42", 42000).get :: Nil)
-    val node_2 = Announcements.makeNodeAnnouncement(randomKey, "node-bob", Color(100.toByte, 200.toByte, 300.toByte), new InetSocketAddress(InetAddress.getByAddress(Array[Byte](192.toByte, 168.toByte, 1.toByte, 42.toByte)), 42000) :: Nil)
+    val node_2 = Announcements.makeNodeAnnouncement(randomKey, "node-bob", Color(100.toByte, 200.toByte, 300.toByte), NodeAddress.fromParts("192.168.1.42", 42000).get :: Nil)
-    val node_3 = Announcements.makeNodeAnnouncement(randomKey, "node-charlie", Color(100.toByte, 200.toByte, 300.toByte), new InetSocketAddress(InetAddress.getByAddress(Array[Byte](192.toByte, 168.toByte, 1.toByte, 42.toByte)), 42000) :: Nil)
+    val node_3 = Announcements.makeNodeAnnouncement(randomKey, "node-charlie", Color(100.toByte, 200.toByte, 300.toByte), NodeAddress.fromParts("192.168.1.42", 42000).get :: Nil)
+    val node_4 = Announcements.makeNodeAnnouncement(randomKey, "node-charlie", Color(100.toByte, 200.toByte, 300.toByte), Tor2("aaaqeayeaudaocaj", 42000) :: Nil)
 
     assert(db.listNodes().toSet === Set.empty)
     db.addNode(node_1)
@@ -52,10 +54,13 @@ class SqliteNetworkDbSpec extends FunSuite {
     assert(db.listNodes().size === 1)
     db.addNode(node_2)
     db.addNode(node_3)
-    assert(db.listNodes().toSet === Set(node_1, node_2, node_3))
+    db.addNode(node_4)
+    assert(db.listNodes().toSet === Set(node_1, node_2, node_3, node_4))
     db.removeNode(node_2.nodeId)
-    assert(db.listNodes().toSet === Set(node_1, node_3))
+    assert(db.listNodes().toSet === Set(node_1, node_3, node_4))
     db.updateNode(node_1)
+
+    assert(node_4.addresses == List(Tor2("aaaqeayeaudaocaj", 42000)))
   }
 
   test("add/remove/list channels and channel_updates") {

eclair-core/src/test/scala/fr/acinq/eclair/db/SqlitePeersDbSpec.scala

@@ -19,8 +19,10 @@ package fr.acinq.eclair.db
 import java.net.{InetAddress, InetSocketAddress}
 import java.sql.DriverManager
 
+import com.google.common.net.HostAndPort
 import fr.acinq.eclair.db.sqlite.SqlitePeersDb
 import fr.acinq.eclair.randomKey
+import fr.acinq.eclair.wire.{NodeAddress, Tor2, Tor3}
 import org.scalatest.FunSuite
 
 
@@ -38,10 +40,10 @@ class SqlitePeersDbSpec extends FunSuite {
     val sqlite = inmem
     val db = new SqlitePeersDb(sqlite)
 
-    val peer_1 = (randomKey.publicKey, new InetSocketAddress(InetAddress.getLoopbackAddress, 1111))
+    val peer_1 = (randomKey.publicKey, NodeAddress.fromParts("127.0.0.1", 42000).get)
-    val peer_1_bis = (peer_1._1, new InetSocketAddress(InetAddress.getLoopbackAddress, 1112))
+    val peer_1_bis = (peer_1._1, NodeAddress.fromParts("127.0.0.1", 1112).get)
-    val peer_2 = (randomKey.publicKey, new InetSocketAddress(InetAddress.getLoopbackAddress, 2222))
+    val peer_2 = (randomKey.publicKey, Tor2("z4zif3fy7fe7bpg3", 4231))
-    val peer_3 = (randomKey.publicKey, new InetSocketAddress(InetAddress.getLoopbackAddress, 3333))
+    val peer_3 = (randomKey.publicKey, Tor3("mrl2d3ilhctt2vw4qzvmz3etzjvpnc6dczliq5chrxetthgbuczuggyd", 4231))
 
     assert(db.listPeers().toSet === Set.empty)
     db.addOrUpdatePeer(peer_1._1, peer_1._2)

eclair-core/src/test/scala/fr/acinq/eclair/integration/IntegrationSpec.scala

@@ -131,7 +131,7 @@ class IntegrationSpec extends TestKit(ActorSystem("test")) with BitcoindService
     val address = node2.nodeParams.publicAddresses.head
     sender.send(node1.switchboard, Peer.Connect(NodeURI(
       nodeId = node2.nodeParams.nodeId,
-      address = HostAndPort.fromParts(address.getHostString, address.getPort))))
+      address = HostAndPort.fromParts(address.socketAddress.getHostString, address.socketAddress.getPort))))
     sender.expectMsgAnyOf(10 seconds, "connected", "already connected")
     sender.send(node1.switchboard, Peer.OpenChannel(
       remoteNodeId = node2.nodeParams.nodeId,
@@ -250,15 +250,20 @@ class IntegrationSpec extends TestKit(ActorSystem("test")) with BitcoindService
     val amountMsat = MilliSatoshi(4200000)
     sender.send(nodes("D").paymentHandler, ReceivePayment(Some(amountMsat), "1 coffee"))
     val pr = sender.expectMsgType[PaymentRequest]
-    // then we make the actual payment
+    // then we make the actual payment, do not randomize the route to make sure we route through node B
-    val sendReq = SendPayment(amountMsat.amount, pr.paymentHash, nodes("D").nodeParams.nodeId)
+    val sendReq = SendPayment(amountMsat.amount, pr.paymentHash, nodes("D").nodeParams.nodeId, randomize = Some(false))
     sender.send(nodes("A").paymentInitiator, sendReq)
     // A will receive an error from B that include the updated channel update, then will retry the payment
     sender.expectMsgType[PaymentSucceeded](5 seconds)
+
+    awaitCond({
       // in the meantime, the router will have updated its state
       sender.send(nodes("A").router, 'updatesMap)
-    assert(sender.expectMsgType[Map[ChannelDesc, ChannelUpdate]].apply(ChannelDesc(channelUpdateBC.shortChannelId, nodes("B").nodeParams.nodeId, nodes("C").nodeParams.nodeId)) === channelUpdateBC)
       // we then put everything back like before by asking B to refresh its channel update (this will override the one we created)
+      val update = sender.expectMsgType[Map[ChannelDesc, ChannelUpdate]](10 seconds).apply(ChannelDesc(channelUpdateBC.shortChannelId, nodes("B").nodeParams.nodeId, nodes("C").nodeParams.nodeId))
+      update == channelUpdateBC
+    }, max = 30 seconds, interval = 1 seconds)
+
     // first let's wait 3 seconds to make sure the timestamp of the new channel_update will be strictly greater than the former
     sender.expectNoMsg(3 seconds)
     sender.send(nodes("B").register, ForwardShortId(shortIdBC, TickRefreshChannelUpdate))
@@ -411,8 +416,9 @@ class IntegrationSpec extends TestKit(ActorSystem("test")) with BitcoindService
     val previouslyReceivedByC = res.filter(_ \ "address" == JString(finalAddressC)).flatMap(_ \ "txids" \\ classOf[JString])
     // we then kill the connection between C and F
     sender.send(nodes("F1").switchboard, 'peers)
-    val peers = sender.expectMsgType[Map[PublicKey, ActorRef]]
+    val peers = sender.expectMsgType[Iterable[ActorRef]]
-    peers(nodes("C").nodeParams.nodeId) ! Disconnect
+    // F's only node is C
+    peers.head ! Disconnect
     // we then wait for F to be in disconnected state
     awaitCond({
       sender.send(nodes("F1").register, Forward(htlc.channelId, CMD_GETSTATE))
@@ -489,8 +495,9 @@ class IntegrationSpec extends TestKit(ActorSystem("test")) with BitcoindService
     val previouslyReceivedByC = res.filter(_ \ "address" == JString(finalAddressC)).flatMap(_ \ "txids" \\ classOf[JString])
     // we then kill the connection between C and F
     sender.send(nodes("F2").switchboard, 'peers)
-    val peers = sender.expectMsgType[Map[PublicKey, ActorRef]]
+    val peers = sender.expectMsgType[Iterable[ActorRef]]
-    peers(nodes("C").nodeParams.nodeId) ! Disconnect
+    // F's only node is C
+    peers.head ! Disconnect
     // we then wait for F to be in disconnected state
     awaitCond({
       sender.send(nodes("F2").register, Forward(htlc.channelId, CMD_GETSTATE))

eclair-core/src/test/scala/fr/acinq/eclair/interop/rustytests/RustyTestsSpec.scala

@@ -21,7 +21,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
 
 import akka.actor.{ActorRef, ActorSystem, Props}
 import akka.testkit.{TestFSMRef, TestKit, TestProbe}
-import fr.acinq.eclair.Globals
+import fr.acinq.eclair.{Globals, TestUtils}
 import fr.acinq.eclair.TestConstants.{Alice, Bob}
 import fr.acinq.eclair.blockchain._
 import fr.acinq.eclair.blockchain.fee.FeeratesPerKw
@@ -75,7 +75,7 @@ class RustyTestsSpec extends TestKit(ActorSystem("test")) with Matchers with fix
       pipe ! new File(getClass.getResource(s"/scenarii/${test.name}.script").getFile)
       latch.await(30, TimeUnit.SECONDS)
       val ref = Source.fromFile(getClass.getResource(s"/scenarii/${test.name}.script.expected").getFile).getLines().filterNot(_.startsWith("#")).toList
-      val res = Source.fromFile(new File(s"${System.getProperty("buildDirectory")}/result.tmp")).getLines().filterNot(_.startsWith("#")).toList
+      val res = Source.fromFile(new File(TestUtils.BUILD_DIRECTORY, "result.tmp")).getLines().filterNot(_.startsWith("#")).toList
       withFixture(test.toNoArgTest(FixtureParam(ref, res)))
     }
   }

eclair-core/src/test/scala/fr/acinq/eclair/interop/rustytests/SynchronizationPipe.scala

@@ -21,6 +21,7 @@ import java.util.concurrent.CountDownLatch
 
 import akka.actor.{Actor, ActorLogging, ActorRef, Stash}
 import fr.acinq.bitcoin.BinaryData
+import fr.acinq.eclair.TestUtils
 import fr.acinq.eclair.channel._
 import fr.acinq.eclair.transactions.{IN, OUT}
 
@@ -48,7 +49,7 @@ class SynchronizationPipe(latch: CountDownLatch) extends Actor with ActorLogging
   val echo = "echo (.*)".r
   val dump = "(.):dump".r
 
-  val fout = new BufferedWriter(new FileWriter(s"${System.getProperty("buildDirectory")}/result.tmp"))
+  val fout = new BufferedWriter(new FileWriter(new File(TestUtils.BUILD_DIRECTORY, "result.tmp")))
 
   def exec(script: List[String], a: ActorRef, b: ActorRef): Unit = {
     def resolve(x: String) = if (x == "A") a else b

eclair-core/src/test/scala/fr/acinq/eclair/io/PeerSpec.scala

@@ -1,9 +1,26 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.io
 
 import java.net.InetSocketAddress
 
 import akka.actor.ActorRef
 import akka.testkit.TestProbe
+import com.google.common.net.HostAndPort
 import fr.acinq.eclair.randomBytes
 import fr.acinq.bitcoin.Crypto.PublicKey
 import fr.acinq.eclair.TestConstants._
@@ -12,7 +29,7 @@ import fr.acinq.eclair.crypto.TransportHandler
 import fr.acinq.eclair.io.Peer.{CHANNELID_ZERO, ResumeAnnouncements, SendPing}
 import fr.acinq.eclair.router.RoutingSyncSpec.makeFakeRoutingInfo
 import fr.acinq.eclair.router.{ChannelRangeQueries, ChannelRangeQueriesSpec, Rebroadcast}
-import fr.acinq.eclair.wire.{Error, Ping, Pong}
+import fr.acinq.eclair.wire.{Error, NodeAddress, Ping, Pong}
 import fr.acinq.eclair.{ShortChannelId, TestkitBaseClass, wire}
 import org.scalatest.Outcome
 
@@ -45,7 +62,7 @@ class PeerSpec extends TestkitBaseClass {
     // let's simulate a connection
     val probe = TestProbe()
     probe.send(peer, Peer.Init(None, Set.empty))
-    authenticator.send(peer, Authenticator.Authenticated(connection.ref, transport.ref, remoteNodeId, InetSocketAddress.createUnresolved("foo.bar", 42000), false, None))
+    authenticator.send(peer, Authenticator.Authenticated(connection.ref, transport.ref, remoteNodeId, new InetSocketAddress("1.2.3.4", 42000), outgoing = true, None))
     transport.expectMsgType[TransportHandler.Listener]
     transport.expectMsgType[wire.Init]
     transport.send(peer, wire.Init(Bob.nodeParams.globalFeatures, Bob.nodeParams.localFeatures))
@@ -168,36 +185,20 @@ class PeerSpec extends TestkitBaseClass {
     }
     transport.expectNoMsg(1 second) // peer hasn't acknowledged the messages
 
-    // now let's assume that the router isn't happy with those channels because the funding tx is not found
+    // now let's assume that the router isn't happy with those channels because the announcement is invalid
-    for (c <- channels) {
+    router.send(peer, Peer.InvalidAnnouncement(channels(0)))
-      router.send(peer, Peer.NonexistingChannel(c))
+    // peer will return a connection-wide error, including the hex-encoded representation of the bad message
-    }
+    val error1 = transport.expectMsgType[Error]
-    // peer will temporary ignore announcements coming from bob
+    assert(error1.channelId === CHANNELID_ZERO)
-    for (ann <- channels ++ updates) {
+    assert(new String(error1.data).startsWith("couldn't verify channel! shortChannelId="))
-      transport.send(peer, ann)
-      transport.expectMsg(TransportHandler.ReadAck(ann))
-    }
-    router.expectNoMsg(1 second)
-    // other routing messages go through
-    transport.send(peer, query)
-    router.expectMsg(Peer.PeerRoutingMessage(transport.ref, remoteNodeId, query))
 
-    // after a while the ban is lifted
-    probe.send(peer, ResumeAnnouncements)
-
-    // and announcements are processed again
-    for (c <- channels) {
-      transport.send(peer, c)
-      router.expectMsg(Peer.PeerRoutingMessage(transport.ref, remoteNodeId, c))
-    }
-    transport.expectNoMsg(1 second) // peer hasn't acknowledged the messages
 
     // let's assume that one of the sigs were invalid
     router.send(peer, Peer.InvalidSignature(channels(0)))
     // peer will return a connection-wide error, including the hex-encoded representation of the bad message
-    val error = transport.expectMsgType[Error]
+    val error2 = transport.expectMsgType[Error]
-    assert(error.channelId === CHANNELID_ZERO)
+    assert(error2.channelId === CHANNELID_ZERO)
-    assert(new String(error.data).startsWith("bad announcement sig! bin=0100"))
+    assert(new String(error2.data).startsWith("bad announcement sig! bin=0100"))
   }
 
 }

eclair-core/src/test/scala/fr/acinq/eclair/payment/ChannelSelectionSpec.scala

@@ -0,0 +1,113 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.payment
+
+import fr.acinq.bitcoin.Block
+import fr.acinq.bitcoin.Crypto.PublicKey
+import fr.acinq.eclair.channel.{CMD_ADD_HTLC, CMD_FAIL_HTLC}
+import fr.acinq.eclair.crypto.Sphinx
+import fr.acinq.eclair.payment.Relayer.{OutgoingChannel, RelayPayload}
+import fr.acinq.eclair.router.Announcements
+import fr.acinq.eclair.wire._
+import fr.acinq.eclair.{ShortChannelId, randomBytes, randomKey}
+import org.scalatest.FunSuite
+
+import scala.collection.mutable
+
+class ChannelSelectionSpec extends FunSuite {
+
+  /**
+    * This is just a simplified helper function with random values for fields we are not using here
+    */
+  def dummyUpdate(shortChannelId: ShortChannelId, cltvExpiryDelta: Int, htlcMinimumMsat: Long, feeBaseMsat: Long, feeProportionalMillionths: Long, htlcMaximumMsat: Long, enable: Boolean = true) =
+    Announcements.makeChannelUpdate(Block.RegtestGenesisBlock.hash, randomKey, randomKey.publicKey, shortChannelId, cltvExpiryDelta, htlcMinimumMsat, feeBaseMsat, feeProportionalMillionths, htlcMaximumMsat, enable)
+
+  test("handle relay") {
+    val relayPayload = RelayPayload(
+      add = UpdateAddHtlc(randomBytes(32), 42, 1000000, randomBytes(32), 70, ""),
+      payload = PerHopPayload(ShortChannelId(12345), amtToForward = 998900, outgoingCltvValue = 60),
+      nextPacket = Sphinx.LAST_PACKET // just a placeholder
+    )
+
+    val channelUpdate = dummyUpdate(ShortChannelId(12345), 10, 100, 1000, 100, 10000000, true)
+
+    implicit val log = akka.event.NoLogging
+
+    // nominal case
+    assert(Relayer.handleRelay(relayPayload, Some(channelUpdate)) === Right(CMD_ADD_HTLC(relayPayload.payload.amtToForward, relayPayload.add.paymentHash, relayPayload.payload.outgoingCltvValue, relayPayload.nextPacket.serialize, upstream_opt = Some(relayPayload.add), commit = true, redirected = false)))
+    // redirected to preferred channel
+    assert(Relayer.handleRelay(relayPayload, Some(channelUpdate.copy(shortChannelId = ShortChannelId(1111)))) === Right(CMD_ADD_HTLC(relayPayload.payload.amtToForward, relayPayload.add.paymentHash, relayPayload.payload.outgoingCltvValue, relayPayload.nextPacket.serialize, upstream_opt = Some(relayPayload.add), commit = true, redirected = true)))
+    // no channel_update
+    assert(Relayer.handleRelay(relayPayload, channelUpdate_opt = None) === Left(CMD_FAIL_HTLC(relayPayload.add.id, Right(UnknownNextPeer), commit = true)))
+    // channel disabled
+    val channelUpdate_disabled = channelUpdate.copy(channelFlags = Announcements.makeChannelFlags(true, enable = false))
+    assert(Relayer.handleRelay(relayPayload, Some(channelUpdate_disabled)) === Left(CMD_FAIL_HTLC(relayPayload.add.id, Right(ChannelDisabled(channelUpdate_disabled.messageFlags, channelUpdate_disabled.channelFlags, channelUpdate_disabled)), commit = true)))
+    // amount too low
+    val relayPayload_toolow = relayPayload.copy(payload = relayPayload.payload.copy(amtToForward = 99))
+    assert(Relayer.handleRelay(relayPayload_toolow, Some(channelUpdate)) === Left(CMD_FAIL_HTLC(relayPayload.add.id, Right(AmountBelowMinimum(relayPayload_toolow.payload.amtToForward, channelUpdate)), commit = true)))
+    // incorrect cltv expiry
+    val relayPayload_incorrectcltv = relayPayload.copy(payload = relayPayload.payload.copy(outgoingCltvValue = 42))
+    assert(Relayer.handleRelay(relayPayload_incorrectcltv, Some(channelUpdate)) === Left(CMD_FAIL_HTLC(relayPayload.add.id, Right(IncorrectCltvExpiry(relayPayload_incorrectcltv.payload.outgoingCltvValue, channelUpdate)), commit = true)))
+    // insufficient fee
+    val relayPayload_insufficientfee = relayPayload.copy(payload = relayPayload.payload.copy(amtToForward = 998910))
+    assert(Relayer.handleRelay(relayPayload_insufficientfee, Some(channelUpdate)) === Left(CMD_FAIL_HTLC(relayPayload.add.id, Right(FeeInsufficient(relayPayload_insufficientfee.add.amountMsat, channelUpdate)), commit = true)))
+    // note that a generous fee is ok!
+    val relayPayload_highfee = relayPayload.copy(payload = relayPayload.payload.copy(amtToForward = 900000))
+    assert(Relayer.handleRelay(relayPayload_highfee, Some(channelUpdate)) === Right(CMD_ADD_HTLC(relayPayload_highfee.payload.amtToForward, relayPayload_highfee.add.paymentHash, relayPayload_highfee.payload.outgoingCltvValue, relayPayload_highfee.nextPacket.serialize, upstream_opt = Some(relayPayload.add), commit = true, redirected = false)))
+  }
+
+  test("relay channel selection") {
+
+    val relayPayload = RelayPayload(
+      add = UpdateAddHtlc(randomBytes(32), 42, 1000000, randomBytes(32), 70, ""),
+      payload = PerHopPayload(ShortChannelId(12345), amtToForward = 998900, outgoingCltvValue = 60),
+      nextPacket = Sphinx.LAST_PACKET // just a placeholder
+    )
+
+    val (a, b) = (randomKey.publicKey, randomKey.publicKey)
+    val channelUpdate = dummyUpdate(ShortChannelId(12345), 10, 100, 1000, 100, 10000000, true)
+
+    val channelUpdates = Map(
+      ShortChannelId(11111) -> OutgoingChannel(a, channelUpdate, 100000000),
+      ShortChannelId(12345) -> OutgoingChannel(a, channelUpdate, 20000000),
+      ShortChannelId(22222) -> OutgoingChannel(a, channelUpdate, 10000000),
+      ShortChannelId(33333) -> OutgoingChannel(a, channelUpdate, 100000),
+      ShortChannelId(44444) -> OutgoingChannel(b, channelUpdate, 1000000)
+    )
+
+    val node2channels = new mutable.HashMap[PublicKey, mutable.Set[ShortChannelId]] with mutable.MultiMap[PublicKey, ShortChannelId]
+    node2channels.put(a, mutable.Set(ShortChannelId(12345), ShortChannelId(11111), ShortChannelId(22222), ShortChannelId(33333)))
+    node2channels.put(b, mutable.Set(ShortChannelId(44444)))
+
+    implicit val log = akka.event.NoLogging
+
+    import com.softwaremill.quicklens._
+
+    // select the channel to the same node, with the lowest balance but still high enough to handle the payment
+    assert(Relayer.selectPreferredChannel(relayPayload, channelUpdates, node2channels) === ShortChannelId(22222))
+    // higher amount payment (have to increased incoming htlc amount for fees to be sufficient)
+    assert(Relayer.selectPreferredChannel(relayPayload.modify(_.add.amountMsat).setTo(60000000).modify(_.payload.amtToForward).setTo(50000000), channelUpdates, node2channels) === ShortChannelId(11111))
+    // lower amount payment
+    assert(Relayer.selectPreferredChannel(relayPayload.modify(_.payload.amtToForward).setTo(1000), channelUpdates, node2channels) === ShortChannelId(33333))
+    // payment too high, no suitable channel, we keep the requested one
+    assert(Relayer.selectPreferredChannel(relayPayload.modify(_.payload.amtToForward).setTo(1000000000), channelUpdates, node2channels) === ShortChannelId(12345))
+    // invalid cltv expiry, no suitable channel, we keep the requested one
+    assert(Relayer.selectPreferredChannel(relayPayload.modify(_.payload.outgoingCltvValue).setTo(40), channelUpdates, node2channels) === ShortChannelId(12345))
+
+  }
+
+}

eclair-core/src/test/scala/fr/acinq/eclair/payment/RelayerSpec.scala

@@ -55,9 +55,11 @@ class RelayerSpec extends TestkitBaseClass {
   val channelId_ab: BinaryData = randomBytes(32)
   val channelId_bc: BinaryData = randomBytes(32)
 
-  def makeCommitments(channelId: BinaryData) = Commitments(null, null, 0.toByte, null,
+  def makeCommitments(channelId: BinaryData) = new Commitments(null, null, 0.toByte, null,
     RemoteCommit(42, CommitmentSpec(Set.empty, 20000, 5000000, 100000000), "00" * 32, randomKey.toPoint),
-    null, null, 0, 0, Map.empty, null, null, null, channelId)
+    null, null, 0, 0, Map.empty, null, null, null, channelId) {
+    override def availableBalanceForSendMsat: Long = remoteCommit.spec.toRemoteMsat // approximation
+  }
 
   test("relay an htlc-add") { f =>
     import f._

eclair-core/src/test/scala/fr/acinq/eclair/router/AnnouncementsBatchValidationSpec.scala

@@ -59,13 +59,13 @@ class AnnouncementsBatchValidationSpec extends FunSuite {
     val sender = TestProbe()
 
     extendedBitcoinClient.validate(announcements(0)).pipeTo(sender.ref)
-    sender.expectMsgType[ValidateResult].tx.isDefined
+    sender.expectMsgType[ValidateResult].fundingTx.isRight
 
     extendedBitcoinClient.validate(announcements(1).copy(shortChannelId = ShortChannelId(Long.MaxValue))).pipeTo(sender.ref) // invalid block height
-    sender.expectMsgType[ValidateResult].tx.isEmpty
+    sender.expectMsgType[ValidateResult].fundingTx.isRight
 
     extendedBitcoinClient.validate(announcements(2).copy(shortChannelId = ShortChannelId(500, 1000, 0))).pipeTo(sender.ref) // invalid tx index
-    sender.expectMsgType[ValidateResult].tx.isEmpty
+    sender.expectMsgType[ValidateResult].fundingTx.isRight
 
   }
 

eclair-core/src/test/scala/fr/acinq/eclair/router/BaseRouterSpec.scala

@@ -22,7 +22,7 @@ import fr.acinq.bitcoin.Crypto.PrivateKey
 import fr.acinq.bitcoin.Script.{pay2wsh, write}
 import fr.acinq.bitcoin.{BinaryData, Block, Satoshi, Transaction, TxOut}
 import fr.acinq.eclair.TestConstants.Alice
-import fr.acinq.eclair.blockchain.{ValidateRequest, ValidateResult, WatchSpentBasic}
+import fr.acinq.eclair.blockchain.{UtxoStatus, ValidateRequest, ValidateResult, WatchSpentBasic}
 import fr.acinq.eclair.io.Peer.PeerRoutingMessage
 import fr.acinq.eclair.router.Announcements._
 import fr.acinq.eclair.transactions.Scripts
@@ -126,10 +126,10 @@ abstract class BaseRouterSpec extends TestkitBaseClass {
       watcher.expectMsg(ValidateRequest(chan_cd))
       watcher.expectMsg(ValidateRequest(chan_ef))
       // and answers with valid scripts
-      watcher.send(router, ValidateResult(chan_ab, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, funding_b)))) :: Nil, lockTime = 0)), true, None))
+      watcher.send(router, ValidateResult(chan_ab, Right((Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, funding_b)))) :: Nil, lockTime = 0), UtxoStatus.Unspent))))
-      watcher.send(router, ValidateResult(chan_bc, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_b, funding_c)))) :: Nil, lockTime = 0)), true, None))
+      watcher.send(router, ValidateResult(chan_bc, Right((Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_b, funding_c)))) :: Nil, lockTime = 0), UtxoStatus.Unspent))))
-      watcher.send(router, ValidateResult(chan_cd, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_c, funding_d)))) :: Nil, lockTime = 0)), true, None))
+      watcher.send(router, ValidateResult(chan_cd, Right((Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_c, funding_d)))) :: Nil, lockTime = 0), UtxoStatus.Unspent))))
-      watcher.send(router, ValidateResult(chan_ef, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_e, funding_f)))) :: Nil, lockTime = 0)), true, None))
+      watcher.send(router, ValidateResult(chan_ef, Right((Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_e, funding_f)))) :: Nil, lockTime = 0), UtxoStatus.Unspent))))
       // watcher receives watch-spent request
       watcher.expectMsgType[WatchSpentBasic]
       watcher.expectMsgType[WatchSpentBasic]

eclair-core/src/test/scala/fr/acinq/eclair/router/ChannelRangeQueriesSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.router
 
 import fr.acinq.bitcoin.Block

eclair-core/src/test/scala/fr/acinq/eclair/router/GraphSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.router
 
 import fr.acinq.bitcoin.Crypto.PublicKey

eclair-core/src/test/scala/fr/acinq/eclair/router/RouteCalculationSpec.scala

@@ -167,15 +167,15 @@ class RouteCalculationSpec extends FunSuite {
 
     val updates = List(
       makeUpdate(1L, f, g, 0, 0),
-      makeUpdate(4L, f, h, 50, 0), // our starting node F has a direct channel with H, no routing fees are paid to traverse that
+      makeUpdate(4L, f, i, 50, 0), // our starting node F has a direct channel with I
       makeUpdate(2L, g, h, 0, 0),
       makeUpdate(3L, h, i, 0, 0)
     ).toMap
 
     val graph = makeGraph(updates)
 
-    val route = Router.findRoute(graph, f, i, DEFAULT_AMOUNT_MSAT, numRoutes = 1)
+    val route = Router.findRoute(graph, f, i, DEFAULT_AMOUNT_MSAT, numRoutes = 2)
-    assert(route.map(hops2Ids) === Success(4 :: 3 :: Nil))
+    assert(route.map(hops2Ids) === Success(4 :: Nil))
   }
 
   test("if there are multiple channels between the same node, select the cheapest") {

eclair-core/src/test/scala/fr/acinq/eclair/router/RouterSpec.scala

@@ -77,10 +77,10 @@ class RouterSpec extends BaseRouterSpec {
     watcher.expectMsg(ValidateRequest(chan_ax))
     watcher.expectMsg(ValidateRequest(chan_ay))
     watcher.expectMsg(ValidateRequest(chan_az))
-    watcher.send(router, ValidateResult(chan_ac, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, funding_c)))) :: Nil, lockTime = 0)), true, None))
+    watcher.send(router, ValidateResult(chan_ac, Right(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, funding_c)))) :: Nil, lockTime = 0), UtxoStatus.Unspent)))
-    watcher.send(router, ValidateResult(chan_ax, None, false, None))
+    watcher.send(router, ValidateResult(chan_ax, Left(new RuntimeException(s"funding tx not found"))))
-    watcher.send(router, ValidateResult(chan_ay, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, randomKey.publicKey)))) :: Nil, lockTime = 0)), true, None))
+    watcher.send(router, ValidateResult(chan_ay, Right(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, randomKey.publicKey)))) :: Nil, lockTime = 0), UtxoStatus.Unspent)))
-    watcher.send(router, ValidateResult(chan_az, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, priv_funding_z.publicKey)))) :: Nil, lockTime = 0)), false, None))
+    watcher.send(router, ValidateResult(chan_az, Right(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, priv_funding_z.publicKey)))) :: Nil, lockTime = 0), UtxoStatus.Spent(spendingTxConfirmed = true))))
     watcher.expectMsgType[WatchSpentBasic]
     watcher.expectNoMsg(1 second)
 
@@ -245,7 +245,7 @@ class RouterSpec extends BaseRouterSpec {
     probe.send(router, PeerRoutingMessage(null, remoteNodeId, announcement))
     watcher.expectMsgType[ValidateRequest]
     probe.send(router, PeerRoutingMessage(null, remoteNodeId, update))
-    watcher.send(router, ValidateResult(announcement, Some(Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, funding_c)))) :: Nil, lockTime = 0)), true, None))
+    watcher.send(router, ValidateResult(announcement, Right((Transaction(version = 0, txIn = Nil, txOut = TxOut(Satoshi(1000000), write(pay2wsh(Scripts.multiSig2of2(funding_a, funding_c)))) :: Nil, lockTime = 0), UtxoStatus.Unspent))))
 
     probe.send(router, TickPruneStaleChannels)
     val sender = TestProbe()

eclair-core/src/test/scala/fr/acinq/eclair/router/RoutingSyncSpec.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package fr.acinq.eclair.router
 
 import akka.actor.ActorSystem

eclair-core/src/test/scala/fr/acinq/eclair/tor/Socks5ConnectionSpec.scala

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.tor
+
+import java.net.InetSocketAddress
+
+import org.scalatest.FunSuite
+
+/**
+  * Created by PM on 27/01/2017.
+  */
+
+class Socks5ConnectionSpec extends FunSuite {
+
+  test("get proxy address") {
+    val proxyAddress = new InetSocketAddress(9050)
+
+    assert(Socks5ProxyParams.proxyAddress(
+      socketAddress = new InetSocketAddress("1.2.3.4", 9735),
+      proxyParams = Socks5ProxyParams(address = proxyAddress, credentials_opt = None, randomizeCredentials = false, useForIPv4 = true, useForIPv6 = true, useForTor = true)) == Some(proxyAddress))
+
+    assert(Socks5ProxyParams.proxyAddress(
+      socketAddress = new InetSocketAddress("1.2.3.4", 9735),
+      proxyParams = Socks5ProxyParams(address = proxyAddress, credentials_opt = None, randomizeCredentials = false, useForIPv4 = false, useForIPv6 = true, useForTor = true)) == None)
+
+    assert(Socks5ProxyParams.proxyAddress(
+      socketAddress = new InetSocketAddress("[fc92:97a3:e057:b290:abd8:9bd6:135d:7e7]", 9735),
+      proxyParams = Socks5ProxyParams(address = proxyAddress, credentials_opt = None, randomizeCredentials = false, useForIPv4 = true, useForIPv6 = true, useForTor = true)) == Some(proxyAddress))
+
+    assert(Socks5ProxyParams.proxyAddress(
+      socketAddress = new InetSocketAddress("[fc92:97a3:e057:b290:abd8:9bd6:135d:7e7]", 9735),
+      proxyParams = Socks5ProxyParams(address = proxyAddress, credentials_opt = None, randomizeCredentials = false, useForIPv4 = true, useForIPv6 = false, useForTor = true)) == None)
+
+    assert(Socks5ProxyParams.proxyAddress(
+      socketAddress = new InetSocketAddress("iq7zhmhck54vcax2vlrdcavq2m32wao7ekh6jyeglmnuuvv3js57r4id.onion", 9735),
+      proxyParams = Socks5ProxyParams(address = proxyAddress, credentials_opt = None, randomizeCredentials = false, useForIPv4 = true, useForIPv6 = true, useForTor = true)) == Some(proxyAddress))
+
+    assert(Socks5ProxyParams.proxyAddress(
+      socketAddress = new InetSocketAddress("iq7zhmhck54vcax2vlrdcavq2m32wao7ekh6jyeglmnuuvv3js57r4id.onion", 9735),
+      proxyParams = Socks5ProxyParams(address = proxyAddress, credentials_opt = None, randomizeCredentials = false, useForIPv4 = true, useForIPv6 = true, useForTor = false)) == None)
+
+
+  }
+
+}

eclair-core/src/test/scala/fr/acinq/eclair/tor/TorProtocolHandlerSpec.scala

@@ -0,0 +1,303 @@
+/*
+ * Copyright 2018 ACINQ SAS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fr.acinq.eclair.tor
+
+import java.net.InetSocketAddress
+import java.nio.file.{Files, Paths}
+
+import akka.actor.ActorSystem
+import akka.io.Tcp.Connected
+import akka.testkit.{ImplicitSender, TestActorRef, TestKit}
+import akka.util.ByteString
+import fr.acinq.bitcoin.BinaryData
+import fr.acinq.eclair.TestUtils
+import fr.acinq.eclair.wire.{NodeAddress, Tor2, Tor3}
+import org.scalatest._
+
+import scala.concurrent.duration._
+import scala.concurrent.{Await, Promise}
+
+class TorProtocolHandlerSpec extends TestKit(ActorSystem("test"))
+  with FunSuiteLike
+  with ImplicitSender
+  with BeforeAndAfterEach
+  with BeforeAndAfterAll {
+
+  import TorProtocolHandler._
+
+  val LocalHost = new InetSocketAddress("localhost", 8888)
+  val PASSWORD = "foobar"
+  val ClientNonce = "8969A7F3C03CD21BFD1CC49DBBD8F398345261B5B66319DF76BB2FDD8D96BCCA"
+  val PkFilePath = Paths.get(TestUtils.BUILD_DIRECTORY, "testtorpk.dat")
+  val CookieFilePath = Paths.get(TestUtils.BUILD_DIRECTORY, "testtorcookie.dat")
+  val AuthCookie = "AA8593C52DF9713CC5FF6A1D0A045B3FADCAE57745B1348A62A6F5F88D940485"
+
+  override protected def beforeEach(): Unit = {
+    super.afterEach()
+    PkFilePath.toFile.delete()
+  }
+
+  ignore("connect to real tor daemon") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    val protocolHandlerProps = TorProtocolHandler.props(
+      version = OnionServiceVersion("v2"),
+      authentication = Password(PASSWORD),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress))
+
+    val controller = TestActorRef(Controller.props(new InetSocketAddress("localhost", 9051), protocolHandlerProps), "tor")
+
+    val address = Await.result(promiseOnionAddress.future, 30 seconds)
+    println(address)
+  }
+
+  test("happy path v2") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    val protocolHandler = TestActorRef(props(
+      version = OnionServiceVersion("v2"),
+      authentication = Password(PASSWORD),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress)))
+
+    protocolHandler ! Connected(LocalHost, LocalHost)
+
+    expectMsg(ByteString("PROTOCOLINFO 1\r\n"))
+    protocolHandler ! ByteString(
+      "250-PROTOCOLINFO 1\r\n" +
+        "250-AUTH METHODS=HASHEDPASSWORD\r\n" +
+        "250-VERSION Tor=\"0.3.3.5\"\r\n" +
+        "250 OK\r\n"
+    )
+
+    expectMsg(ByteString(s"""AUTHENTICATE "$PASSWORD"\r\n"""))
+    protocolHandler ! ByteString(
+      "250 OK\r\n"
+    )
+
+    expectMsg(ByteString("ADD_ONION NEW:RSA1024 Port=9999,9999\r\n"))
+    protocolHandler ! ByteString(
+      "250-ServiceID=z4zif3fy7fe7bpg3\r\n" +
+        "250-PrivateKey=RSA1024:private-key\r\n" +
+        "250 OK\r\n"
+    )
+    protocolHandler ! GetOnionAddress
+    expectMsg(Some(Tor2("z4zif3fy7fe7bpg3", 9999)))
+
+    val address = Await.result(promiseOnionAddress.future, 3 seconds)
+    assert(address === Tor2("z4zif3fy7fe7bpg3", 9999))
+
+    assert(readString(PkFilePath) === "RSA1024:private-key")
+  }
+
+  test("happy path v3") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    val protocolHandler = TestActorRef(props(
+      version = OnionServiceVersion("v3"),
+      authentication = Password(PASSWORD),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress)))
+
+    protocolHandler ! Connected(LocalHost, LocalHost)
+
+    expectMsg(ByteString("PROTOCOLINFO 1\r\n"))
+    protocolHandler ! ByteString(
+      "250-PROTOCOLINFO 1\r\n" +
+        "250-AUTH METHODS=HASHEDPASSWORD\r\n" +
+        "250-VERSION Tor=\"0.3.4.8\"\r\n" +
+        "250 OK\r\n"
+    )
+
+    expectMsg(ByteString(s"""AUTHENTICATE "$PASSWORD"\r\n"""))
+    protocolHandler ! ByteString(
+      "250 OK\r\n"
+    )
+
+    expectMsg(ByteString("ADD_ONION NEW:ED25519-V3 Port=9999,9999\r\n"))
+    protocolHandler ! ByteString(
+      "250-ServiceID=mrl2d3ilhctt2vw4qzvmz3etzjvpnc6dczliq5chrxetthgbuczuggyd\r\n" +
+        "250-PrivateKey=ED25519-V3:private-key\r\n" +
+        "250 OK\r\n"
+    )
+
+    protocolHandler ! GetOnionAddress
+    expectMsg(Some(Tor3("mrl2d3ilhctt2vw4qzvmz3etzjvpnc6dczliq5chrxetthgbuczuggyd", 9999)))
+
+    val address = Await.result(promiseOnionAddress.future, 3 seconds)
+    assert(address === Tor3("mrl2d3ilhctt2vw4qzvmz3etzjvpnc6dczliq5chrxetthgbuczuggyd", 9999))
+
+    assert(readString(PkFilePath) === "ED25519-V3:private-key")
+  }
+
+  test("v2/v3 compatibility check against tor version") {
+    assert(OnionServiceVersion.isCompatible(V3, "0.3.3.6"))
+    assert(!OnionServiceVersion.isCompatible(V3, "0.3.3.5"))
+    assert(OnionServiceVersion.isCompatible(V3, "0.3.3.6-devel"))
+    assert(OnionServiceVersion.isCompatible(V3, "0.4"))
+    assert(!OnionServiceVersion.isCompatible(V3, "0.2"))
+    assert(OnionServiceVersion.isCompatible(V3, "0.5.1.2.3.4"))
+
+  }
+
+  test("authentication method errors") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    val protocolHandler = TestActorRef(props(
+      version = OnionServiceVersion("v2"),
+      authentication = Password(PASSWORD),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress)))
+
+    protocolHandler ! Connected(LocalHost, LocalHost)
+
+    expectMsg(ByteString("PROTOCOLINFO 1\r\n"))
+    protocolHandler ! ByteString(
+      "250-PROTOCOLINFO 1\r\n" +
+        "250-AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"" + CookieFilePath + "\"\r\n" +
+        "250-VERSION Tor=\"0.3.3.5\"\r\n" +
+        "250 OK\r\n"
+    )
+
+    assert(intercept[TorException] {
+      Await.result(promiseOnionAddress.future, 3 seconds)
+    } === TorException("cannot use authentication 'password', supported methods are 'COOKIE,SAFECOOKIE'"))
+  }
+
+  test("invalid server hash") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    Files.write(CookieFilePath, fr.acinq.eclair.randomBytes(32))
+
+    val protocolHandler = TestActorRef(props(
+      version = OnionServiceVersion("v2"),
+      authentication = SafeCookie(ClientNonce),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress)))
+
+    protocolHandler ! Connected(LocalHost, LocalHost)
+
+    expectMsg(ByteString("PROTOCOLINFO 1\r\n"))
+    protocolHandler ! ByteString(
+      "250-PROTOCOLINFO 1\r\n" +
+        "250-AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"" + CookieFilePath + "\"\r\n" +
+        "250-VERSION Tor=\"0.3.3.5\"\r\n" +
+        "250 OK\r\n"
+    )
+
+    expectMsg(ByteString("AUTHCHALLENGE SAFECOOKIE 8969a7f3c03cd21bfd1cc49dbbd8f398345261b5b66319df76bb2fdd8d96bcca\r\n"))
+    protocolHandler ! ByteString(
+      "250 AUTHCHALLENGE SERVERHASH=6828e74049924f37cbc61f2aad4dd78d8dc09bef1b4c3bf6ff454016ed9d50df SERVERNONCE=b4aa04b6e7e2df60dcb0f62c264903346e05d1675e77795529e22ca90918dee7\r\n"
+    )
+
+    assert(intercept[TorException] {
+      Await.result(promiseOnionAddress.future, 3 seconds)
+    } === TorException("unexpected server hash"))
+  }
+
+
+  test("AUTHENTICATE failure") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    Files.write(CookieFilePath, BinaryData(AuthCookie))
+
+    val protocolHandler = TestActorRef(props(
+      version = OnionServiceVersion("v2"),
+      authentication = SafeCookie(ClientNonce),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress)))
+
+    protocolHandler ! Connected(LocalHost, LocalHost)
+
+    expectMsg(ByteString("PROTOCOLINFO 1\r\n"))
+    protocolHandler ! ByteString(
+      "250-PROTOCOLINFO 1\r\n" +
+        "250-AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"" + CookieFilePath + "\"\r\n" +
+        "250-VERSION Tor=\"0.3.3.5\"\r\n" +
+        "250 OK\r\n"
+    )
+
+    expectMsg(ByteString("AUTHCHALLENGE SAFECOOKIE 8969a7f3c03cd21bfd1cc49dbbd8f398345261b5b66319df76bb2fdd8d96bcca\r\n"))
+    protocolHandler ! ByteString(
+      "250 AUTHCHALLENGE SERVERHASH=6828e74049924f37cbc61f2aad4dd78d8dc09bef1b4c3bf6ff454016ed9d50df SERVERNONCE=b4aa04b6e7e2df60dcb0f62c264903346e05d1675e77795529e22ca90918dee7\r\n"
+    )
+
+    expectMsg(ByteString("AUTHENTICATE 0ddcab5deb39876cdef7af7860a1c738953395349f43b99f4e5e0f131b0515df\r\n"))
+    protocolHandler ! ByteString(
+      "515 Authentication failed: Safe cookie response did not match expected value.\r\n"
+    )
+
+    assert(intercept[TorException] {
+      Await.result(promiseOnionAddress.future, 3 seconds)
+    } === TorException("server returned error: 515 Authentication failed: Safe cookie response did not match expected value."))
+  }
+
+  test("ADD_ONION failure") {
+    val promiseOnionAddress = Promise[NodeAddress]()
+
+    Files.write(CookieFilePath, BinaryData(AuthCookie))
+
+    val protocolHandler = TestActorRef(props(
+      version = OnionServiceVersion("v2"),
+      authentication = SafeCookie(ClientNonce),
+      privateKeyPath = PkFilePath,
+      virtualPort = 9999,
+      onionAdded = Some(promiseOnionAddress)))
+
+    protocolHandler ! Connected(LocalHost, LocalHost)
+
+    expectMsg(ByteString("PROTOCOLINFO 1\r\n"))
+    protocolHandler ! ByteString(
+      "250-PROTOCOLINFO 1\r\n" +
+        "250-AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"" + CookieFilePath + "\"\r\n" +
+        "250-VERSION Tor=\"0.3.3.5\"\r\n" +
+        "250 OK\r\n"
+    )
+
+    expectMsg(ByteString("AUTHCHALLENGE SAFECOOKIE 8969a7f3c03cd21bfd1cc49dbbd8f398345261b5b66319df76bb2fdd8d96bcca\r\n"))
+    protocolHandler ! ByteString(
+      "250 AUTHCHALLENGE SERVERHASH=6828e74049924f37cbc61f2aad4dd78d8dc09bef1b4c3bf6ff454016ed9d50df SERVERNONCE=b4aa04b6e7e2df60dcb0f62c264903346e05d1675e77795529e22ca90918dee7\r\n"
+    )
+
+    expectMsg(ByteString("AUTHENTICATE 0ddcab5deb39876cdef7af7860a1c738953395349f43b99f4e5e0f131b0515df\r\n"))
+    protocolHandler ! ByteString(
+      "250 OK\r\n"
+    )
+
+    expectMsg(ByteString("ADD_ONION NEW:RSA1024 Port=9999,9999\r\n"))
+    protocolHandler ! ByteString(
+      "513 Invalid argument\r\n"
+    )
+
+    val t = intercept[TorException] {
+      Await.result(promiseOnionAddress.future, 3 seconds)
+    }
+
+    assert(intercept[TorException] {
+      Await.result(promiseOnionAddress.future, 3 seconds)
+    } === TorException("server returned error: 513 Invalid argument"))
+  }
+
+
+}

eclair-core/src/test/scala/fr/acinq/eclair/wire/LightningMessageCodecsSpec.scala

@@ -101,6 +101,20 @@ class LightningMessageCodecsSpec extends FunSuite {
       val nodeaddr2 = nodeaddress.decode(bin).require.value
       assert(nodeaddr === nodeaddr2)
     }
+    {
+      val nodeaddr = Tor2("z4zif3fy7fe7bpg3", 4231)
+      val bin = nodeaddress.encode(nodeaddr).require
+      assert(bin === hex"03 cf3282ecb8f949f0bcdb 1087".toBitVector)
+      val nodeaddr2 = nodeaddress.decode(bin).require.value
+      assert(nodeaddr === nodeaddr2)
+    }
+    {
+      val nodeaddr = Tor3("mrl2d3ilhctt2vw4qzvmz3etzjvpnc6dczliq5chrxetthgbuczuggyd", 4231)
+      val bin = nodeaddress.encode(nodeaddr).require
+      assert(bin === hex"04 6457a1ed0b38a73d56dc866accec93ca6af68bc316568874478dc9399cc1a0b3431b03 1087".toBitVector)
+      val nodeaddr2 = nodeaddress.decode(bin).require.value
+      assert(nodeaddr === nodeaddr2)
+    }
   }
 
   test("encode/decode with signature codec") {

eclair-node-gui/src/main/scala/fr/acinq/eclair/gui/controllers/MainController.scala

@@ -195,7 +195,7 @@ class MainController(val handlers: Handlers, val hostServices: HostServices) ext
     })
     networkNodesIPColumn.setCellValueFactory(new Callback[CellDataFeatures[NodeAnnouncement, String], ObservableValue[String]]() {
       def call(pn: CellDataFeatures[NodeAnnouncement, String]) = {
-        val address = pn.getValue.socketAddresses.map(a => HostAndPort.fromParts(a.getHostString, a.getPort)).mkString(",")
+        val address = pn.getValue.addresses.map(a => HostAndPort.fromParts(a.socketAddress.getHostString, a.socketAddress.getPort)).mkString(",")
         new SimpleStringProperty(address)
       }
     })
@@ -365,7 +365,7 @@ class MainController(val handlers: Handlers, val hostServices: HostServices) ext
     bitcoinChain.getStyleClass.add(setup.chain)
 
     val nodeURI_opt = setup.nodeParams.publicAddresses.headOption.map(address => {
-      s"${setup.nodeParams.nodeId}@${HostAndPort.fromParts(address.getHostString, address.getPort)}"
+      s"${setup.nodeParams.nodeId}@${HostAndPort.fromParts(address.socketAddress.getHostString, address.socketAddress.getPort)}"
     })
 
     contextMenu = ContextMenuUtils.buildCopyContext(List(CopyAction("Copy Pubkey", setup.nodeParams.nodeId.toString())))
@@ -456,8 +456,8 @@ class MainController(val handlers: Handlers, val hostServices: HostServices) ext
     copyURI.setOnAction(new EventHandler[ActionEvent] {
       override def handle(event: ActionEvent): Unit = Option(row.getItem) match {
         case Some(pn) => ContextMenuUtils.copyToClipboard(
-          pn.socketAddresses.headOption match {
+          pn.addresses.headOption match {
-            case Some(firstAddress) => s"${pn.nodeId.toString}@${HostAndPort.fromParts(firstAddress.getHostString, firstAddress.getPort)}"
+            case Some(firstAddress) => s"${pn.nodeId.toString}@${HostAndPort.fromParts(firstAddress.socketAddress.getHostString, firstAddress.socketAddress.getPort)}"
             case None => "no URI Known"
           })
         case None =>