Generate bloom filters from wallet (#501)

* Add findAll() to CRUD

* Add getAddressInfo to Wallet API

* Insert pubkey into bloomfilter also inserts hash

In this commit we also simplify parts of
BloomFilterTest, and move what used to be in
BloomFilterSpec into BloomFilterTest.

* Add bloom filter generation to wallet

* Add bloom false positive rate to wallet config

* Add logging to SQL errors

* Fix a bug where HDPath.next returned the wrong path

* Add FutureUtil.sequentially

* Make bloom filter size calculation more explicit

* Return Vector of pubkeys instead of Seq

* Verbose handling of address fetching in test

core-test/src/test/scala/org/bitcoins/core/hd/HDPathTest.scala

@@ -13,9 +13,6 @@ import org.bitcoins.core.protocol.script.WitnessScriptPubKey
 
 class HDPathTest extends BitcoinSUnitTest {
 
-  override implicit val generatorDrivenConfig: PropertyCheckConfiguration =
-    generatorDrivenConfigNewCode
-
   behavior of "HDAccount"
 
   it must "fail to make accounts with negative indices" in {
@@ -79,6 +76,16 @@ class HDPathTest extends BitcoinSUnitTest {
 
   behavior of "HDPath"
 
+  it must "generate the next path" in {
+    forAll(HDGenerators.hdPath) { path =>
+      val next = path.next
+      assert(next != path)
+      // all elements except the last one should be the same
+      assert(next.path.init == path.path.init)
+      assert(next.address.index == path.address.index + 1)
+    }
+  }
+
   it must "have toString/fromString symmetry" in {
     forAll(HDGenerators.hdPath) { path =>
       val pathFromString = HDPath.fromString(path.toString)

core/src/main/scala/org/bitcoins/core/bloom/BloomFilter.scala

@@ -20,12 +20,14 @@ import scodec.bits.{BitVector, ByteVector}
 
 import scala.annotation.tailrec
 import scala.util.hashing.MurmurHash3
+import org.bitcoins.core.crypto.ECPublicKey
+import org.bitcoins.core.util.CryptoUtil
 
 /**
-  * Created by chris on 8/2/16.
-  * Implements a bloom fitler that abides by the semantics of
-  * [[https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki BIP37]].
-  * [[https://github.com/bitcoin/bitcoin/blob/master/src/bloom.h]]
+  * Implements a bloom filter that abides by the semantics of BIP37
+
+  * @see [[https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki BIP37]].
+  * @see [[https://github.com/bitcoin/bitcoin/blob/master/src/bloom.h Bitcoin Core bloom.h]]
   */
 sealed abstract class BloomFilter extends NetworkElement {
 
@@ -44,7 +46,7 @@ sealed abstract class BloomFilter extends NetworkElement {
   /**
     * A set of flags that control how outpoints corresponding to a matched pubkey script are added to the filter.
     * See the 'Comparing Transaction Elements to a Bloom Filter' section in this
-    * [[https://bitcoin.org/en/developer-reference#filterload link]]
+    * @see [[https://bitcoin.org/en/developer-reference#filterload link]]
     */
   def flags: BloomFlag
 
@@ -73,20 +75,43 @@ sealed abstract class BloomFilter extends NetworkElement {
     BloomFilter(filterSize, newData, hashFuncs, tweak, flags)
   }
 
-  /** Inserts a [[org.bitcoins.core.crypto.HashDigest HashDigest]] into `data` */
+  /** Inserts a hash into the filter */
   def insert(hash: HashDigest): BloomFilter = insert(hash.bytes)
 
-  /** Inserts a sequence of [[org.bitcoins.core.crypto.HashDigest HashDigest]]'s into our BloomFilter */
+  /** Inserts a sequence of hashes into the filter */
   def insertHashes(hashes: Seq[HashDigest]): BloomFilter = {
     val byteVectors = hashes.map(_.bytes)
     insertByteVectors(byteVectors)
   }
 
-  /** Inserts a [[org.bitcoins.core.protocol.transaction.TransactionOutPoint TransactionOutPoint]] into `data` */
+  /** Inserts a transaction outpoint into the filter */
   def insert(outPoint: TransactionOutPoint): BloomFilter =
     insert(outPoint.bytes)
 
-  /** Checks if `data` contains the given sequence of bytes */
+  /**
+    * Inserts a public key and it's corresponding hash into the bloom filter
+    *
+    * @note The rationale for inserting both the pubkey and its hash is that
+    *       in most cases where you have an "interesting pubkey" that you
+    *       want to track on the P2P network what you really need to do is
+    *       insert the hash of the public key, as that is what appears on
+    *       the blockchain and that nodes you query with bloom filters will
+    *       see.
+    *
+    * @see  The
+    *       [[https://github.com/bitcoinj/bitcoinj/blob/806afa04419ebdc3c15d5adf065979aa7303e7f6/core/src/main/java/org/bitcoinj/core/BloomFilter.java#L244 BitcoinJ]]
+    *       implementation of this function
+    */
+  def insert(pubkey: ECPublicKey): BloomFilter = {
+    val pubkeyBytes = pubkey.bytes
+    val hash = CryptoUtil.sha256Hash160(pubkeyBytes)
+    insert(pubkeyBytes).insert(hash)
+  }
+
+  /** Checks if the filter contains the given public key */
+  def contains(pubkey: ECPublicKey): Boolean = contains(pubkey.bytes)
+
+  /** Checks if the filter contains the given bytes */
   def contains(bytes: ByteVector): Boolean = {
     val bitIndexes = (0 until hashFuncs.toInt).map(i => murmurHash(i, bytes))
     @tailrec
@@ -290,7 +315,29 @@ object BloomFilter extends Factory[BloomFilter] {
   /**
     * Creates a bloom filter based on the number of elements to be inserted into the filter
     * and the desired false positive rate.
-    * [[https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki#bloom-filter-format BIP37]]
+    *
+    * @see [[https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki#bloom-filter-format BIP37]]
+    */
+  // todo provide default flag?
+  def apply(
+      numElements: Int,
+      falsePositiveRate: Double,
+      flags: BloomFlag): BloomFilter = {
+    val random = Math.floor(Math.random() * UInt32.max.toLong).toLong
+    val tweak = UInt32(random)
+    apply(numElements, falsePositiveRate, tweak, flags)
+  }
+
+  // todo: provide a apply method where you can pass in bloom-able filters
+  // through a type class
+
+  /**
+    * Creates a bloom filter based on the number of elements to be inserted into the filter
+    * and the desired false positive rate.
+    *
+    * @see [[https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki#bloom-filter-format BIP37]]
+    *
+    * @param tweak A random value that only acts to randomize the filter and increase privacy
     */
   def apply(
       numElements: Int,

core/src/main/scala/org/bitcoins/core/hd/HDPath.scala

@@ -38,7 +38,7 @@ private[bitcoins] trait HDPath extends BIP32Path {
     * [[org.bitcoins.core.crypto.ExtKey ExtKey]]
     */
   def next: NextPath =
-    HDAddress(chain, account.index + 1).toPath.asInstanceOf[NextPath]
+    HDAddress(chain, address.index + 1).toPath.asInstanceOf[NextPath]
 
   def account: HDAccount = address.account
 

core/src/main/scala/org/bitcoins/core/util/FutureUtil.scala

@@ -1,8 +1,26 @@
 package org.bitcoins.core.util
 
 import scala.concurrent.Future
+import scala.concurrent.ExecutionContext
 
 object FutureUtil {
 
+  /**
+    * Executes a series of futures sequentially
+    *
+    * @param items The elements we want to transform into futures
+    * @param fun A function that transforms each element into a future
+    * @return The processed elements
+    */
+  def sequentially[T, U](items: Seq[T])(fun: T => Future[U])(
+      implicit ec: ExecutionContext): Future[List[U]] = {
+    val init = Future.successful(List.empty[U])
+    items.foldLeft(init) { (f, item) =>
+      f.flatMap { x =>
+        fun(item).map(_ :: x)
+      }
+    } map (_.reverse)
+  }
+
   val unit: Future[Unit] = Future.successful(())
 }

db-commons/src/main/resources/reference.conf

@@ -5,5 +5,7 @@ bitcoin-s {
     # settings for wallet module
     wallet {
         defaultAccountType = legacy # legacy, segwit, nested-segwit
+
+        bloomFalsePositiveRate = 0.0001 # percentage
     }
 }

db-commons/src/main/scala/org/bitcoins/db/CRUD.scala

@@ -4,6 +4,8 @@ import org.bitcoins.core.util.BitcoinSLogger
 import slick.jdbc.SQLiteProfile.api._
 
 import scala.concurrent.{ExecutionContext, Future}
+import java.sql.SQLException
+import org.bitcoins.core.config.MainNet
 
 /**
   * Created by chris on 9/8/16.
@@ -122,6 +124,10 @@ abstract class CRUD[T, PrimaryKeyType] extends BitcoinSLogger {
 
   protected def findAll(ts: Vector[T]): Query[Table[_], T, Seq]
 
+  /** Finds all elements in the table */
+  def findAll(): Future[Vector[T]] =
+    database.run(table.result).map(_.toVector)
+
 }
 
 case class SafeDatabase(config: AppConfig) extends BitcoinSLogger {
@@ -135,16 +141,28 @@ case class SafeDatabase(config: AppConfig) extends BitcoinSLogger {
     */
   private val foreignKeysPragma = sqlu"PRAGMA foreign_keys = TRUE;"
 
-  def run[R](action: DBIOAction[R, NoStream, _]): Future[R] = {
+  /** Logs the given action and error, if we are not on mainnet */
+  private def logAndThrowError(
+      action: DBIOAction[_, NoStream, _]): PartialFunction[Throwable, Nothing] = {
+    case err: SQLException =>
+      if (config.network != MainNet) {
+        logger.error(
+          s"Error when executing query ${action.getDumpInfo.getNamePlusMainInfo}")
+        logger.error(s"$err")
+      }
+      throw err
+  }
 
+  def run[R](action: DBIOAction[R, NoStream, _])(
+      implicit ec: ExecutionContext): Future[R] = {
     val result = database.run[R](foreignKeysPragma >> action)
-    result
+    result.recoverWith { logAndThrowError(action) }
   }
 
   def runVec[R](action: DBIOAction[Seq[R], NoStream, _])(
       implicit ec: ExecutionContext): Future[Vector[R]] = {
     val result = database.run[Seq[R]](foreignKeysPragma >> action)
-    result.map(_.toVector)
+    result.map(_.toVector).recoverWith { logAndThrowError(action) }
   }
 }
 

db-commons/src/main/scala/org/bitcoins/db/SlickUtil.scala

@@ -2,6 +2,7 @@ package org.bitcoins.db
 
 import scala.concurrent.Future
 import slick.jdbc.SQLiteProfile.api._
+import scala.concurrent.ExecutionContext
 
 sealed abstract class SlickUtil {
 
@@ -9,7 +10,8 @@ sealed abstract class SlickUtil {
   def createAllNoAutoInc[T, U <: Table[T]](
       ts: Vector[T],
       database: SafeDatabase,
-      table: TableQuery[U]): Future[Vector[T]] = {
+      table: TableQuery[U])(
+      implicit ec: ExecutionContext): Future[Vector[T]] = {
     val actions = ts.map(t => (table += t).andThen(DBIO.successful(t)))
     val result = database.run(DBIO.sequence(actions))
     result

wallet-test/src/test/scala/org/bitcoins/wallet/WalletUnitTest.scala

@@ -8,6 +8,15 @@ import org.bitcoins.wallet.api.UnlockWalletError.JsonParsingError
 import org.bitcoins.wallet.api.UnlockWalletSuccess
 import org.bitcoins.core.crypto.AesPassword
 import org.bitcoins.wallet.api.UnlockWalletError.MnemonicNotFound
+import scala.concurrent.Future
+import org.bitcoins.core.util.FutureUtil
+import org.scalatest.compatible.Assertion
+import org.bitcoins.core.hd.HDChainType
+import org.bitcoins.core.hd.HDChainType.Change
+import org.bitcoins.core.hd.HDChainType.External
+import org.bitcoins.core.protocol.BitcoinAddress
+import org.bitcoins.wallet.models.AddressDb
+import org.bitcoins.core.hd.HDChain
 
 class WalletUnitTest extends BitcoinSWalletTest {
 
@@ -40,6 +49,92 @@ class WalletUnitTest extends BitcoinSWalletTest {
     }
   }
 
+  it should "know what the last address index is" in { walletApi =>
+    val wallet = walletApi.asInstanceOf[Wallet]
+
+    def getMostRecent(
+        chain: HDChainType,
+        acctIndex: Int
+    ): Future[AddressDb] = {
+      val recentOptFut: Future[Option[AddressDb]] = chain match {
+        case Change   => wallet.addressDAO.findMostRecentChange(acctIndex)
+        case External => wallet.addressDAO.findMostRecentExternal(acctIndex)
+      }
+
+      recentOptFut.map {
+        case Some(addr) => addr
+        case None       => fail(s"Did not find $chain address!")
+      }
+    }
+
+    def assertIndexIs(
+        chain: HDChainType,
+        addrIndex: Int,
+        accountIndex: Int): Future[Assertion] = {
+      getMostRecent(chain, accountIndex) map { addr =>
+        assert(addr.path.address.index == addrIndex)
+      }
+    }
+
+    val addressesToGenerate = 10
+    val addrRange = 0 to addressesToGenerate
+
+    /**
+      * Generate some addresse, and verify that the correct address index is
+      * being reported
+      */
+    def testChain(chain: HDChainType, accIdx: Int): Future[Assertion] = {
+      val getAddrFunc: () => Future[BitcoinAddress] = chain match {
+        case Change   => wallet.getNewChangeAddress _
+        case External => wallet.getNewAddress _
+      }
+      for {
+        _ <- {
+          val addrF = chain match {
+            case Change   => wallet.addressDAO.findMostRecentChange(accIdx)
+            case External => wallet.addressDAO.findMostRecentExternal(accIdx)
+          }
+          addrF.map {
+            case Some(addr) =>
+              fail(
+                s"Found ${addr.address} $chain address although there was no previous addreses generated")
+            case None =>
+          }
+        }
+        _ <- FutureUtil.sequentially(addrRange)(_ => getAddrFunc())
+        _ <- assertIndexIs(chain,
+                           accountIndex = accIdx,
+                           addrIndex = addressesToGenerate)
+        newest <- getAddrFunc()
+        res <- getMostRecent(chain, accIdx).map { found =>
+          assert(found.address == newest)
+          assert(found.path.address.index == addressesToGenerate + 1)
+        }
+      } yield res
+    }
+
+    for {
+      account <- wallet.getDefaultAccount()
+      accIdx = account.hdAccount.index
+      _ <- testChain(External, accIdx)
+      res <- testChain(Change, accIdx)
+    } yield res
+  }
+
+  it should "generate a bloom filter" in { walletApi: UnlockedWalletApi =>
+    val wallet = walletApi.asInstanceOf[Wallet]
+    for {
+      _ <- FutureUtil.sequentially(0 until 10)(_ => wallet.getNewAddress())
+      bloom <- wallet.getBloomFilter()
+      pubkeys <- wallet.listPubkeys()
+    } yield {
+      pubkeys.foldLeft(succeed) { (_, pub) =>
+        assert(bloom.contains(pub))
+      }
+    }
+
+  }
+
   it should "lock and unlock the wallet" in { wallet: UnlockedWalletApi =>
     val passphrase = wallet.passphrase
     val locked = wallet.lock()

wallet/src/main/scala/org/bitcoins/wallet/api/AddressInfo.scala

@@ -0,0 +1,14 @@
+package org.bitcoins.wallet.api
+
+import org.bitcoins.core.crypto.ECPublicKey
+import org.bitcoins.core.hd.HDPath
+import org.bitcoins.core.config.NetworkParameters
+
+/**
+  * This class represents the result of querying for address info
+  * from our wallet
+  */
+case class AddressInfo(
+    pubkey: ECPublicKey,
+    network: NetworkParameters,
+    path: HDPath)

wallet/src/main/scala/org/bitcoins/wallet/api/WalletApi.scala

@@ -15,6 +15,7 @@ import org.bitcoins.wallet.models.{AccountDb, AddressDb, UTXOSpendingInfoDb}
 import scala.concurrent.Future
 import scala.concurrent.ExecutionContext
 import org.bitcoins.wallet.config.WalletAppConfig
+import org.bitcoins.core.bloom.BloomFilter
 
 /**
   * API for the wallet project.
@@ -38,6 +39,12 @@ sealed trait WalletApi {
   */
 trait LockedWalletApi extends WalletApi {
 
+  /**
+    * Retrieves a bloom filter that that can be sent to a P2P network node
+    * to get information about our transactions, pubkeys and scripts.
+    */
+  def getBloomFilter(): Future[BloomFilter]
+
   /**
     * Adds the provided UTXO to the wallet, making it
     * available for spending.
@@ -81,6 +88,14 @@ trait LockedWalletApi extends WalletApi {
     } yield address
   }
 
+  /**
+    * Mimics the `getaddressinfo` RPC call in Bitcoin Core
+    *
+    * @return If the address is found in our database `Some(address)`
+    *         is returned, otherwise `None`
+    */
+  def getAddressInfo(address: BitcoinAddress): Future[Option[AddressInfo]]
+
   /** Generates a new change address */
   protected[wallet] def getNewChangeAddress(
       account: AccountDb): Future[BitcoinAddress]

wallet/src/main/scala/org/bitcoins/wallet/config/WalletAppConfig.scala

@@ -27,6 +27,9 @@ case class WalletAppConfig(private val conf: Config*) extends AppConfig {
         throw new RuntimeException(s"$other is not a valid account type!")
     }
 
+  lazy val bloomFalsePositiveRate: Double =
+    config.getDouble("wallet.bloomFalsePositiveRate")
+
   override def initialize()(implicit ec: ExecutionContext): Future[Unit] = {
     logger.debug(s"Initializing wallet setup")
 

wallet/src/main/scala/org/bitcoins/wallet/models/AccountDAO.scala

@@ -38,8 +38,4 @@ case class AccountDAO()(
     findByPrimaryKeys(
       accounts.map(acc => (acc.hdAccount.coin, acc.hdAccount.index)))
 
-  def findAll(): Future[Vector[AccountDb]] = {
-    val query = table.result
-    database.run(query).map(_.toVector)
-  }
 }

wallet/src/main/scala/org/bitcoins/wallet/models/AddressDAO.scala

@@ -10,6 +10,7 @@ import slick.sql.SqlAction
 import scala.concurrent.{ExecutionContext, Future}
 import org.bitcoins.core.hd.HDChainType
 import org.bitcoins.wallet.config.WalletAppConfig
+import org.bitcoins.core.crypto.ECPublicKey
 
 case class AddressDAO()(
     implicit val ec: ExecutionContext,
@@ -35,21 +36,25 @@ case class AddressDAO()(
     database.run(query).map(_.headOption)
   }
 
-  def findAll(): Future[Vector[AddressDb]] = {
-    val query = table.result
-    database.run(query).map(_.toVector)
-  }
-
   private def addressesForAccountQuery(
       accountIndex: Int): Query[AddressTable, AddressDb, Seq] =
     table.filter(_.accountIndex === accountIndex)
 
+  /**
+    * Finds the most recent change address in the wallet, if any
+    */
   def findMostRecentChange(accountIndex: Int): Future[Option[AddressDb]] = {
     val query = findMostRecentForChain(accountIndex, HDChainType.Change)
 
     database.run(query)
   }
 
+  /** Finds all public keys in the wallet */
+  def findAllPubkeys(): Future[Vector[ECPublicKey]] = {
+    val query = table.map(_.ecPublicKey).distinct
+    database.run(query.result).map(_.toVector)
+  }
+
   private def findMostRecentForChain(
       accountIndex: Int,
       chain: HDChainType): SqlAction[Option[AddressDb], NoStream, Effect.Read] = {
@@ -61,6 +66,9 @@ case class AddressDAO()(
       .headOption
   }
 
+  /**
+    * Finds the most recent external address in the wallet, if any
+    */
   def findMostRecentExternal(accountIndex: Int): Future[Option[AddressDb]] = {
     val query = findMostRecentForChain(accountIndex, HDChainType.External)
     database.run(query)