Implement getBestFilterHeader based on a number of block headers that… (#1926)

* Implement getBestFilterHeader based on a number of block headers that can be passed in as a parameter. These headers can be used to indicate what your current best chain is * Bring back compiler opts * Fix compiler error for maxByOption as it isn't in the 2.12 std library * Implement a context free best filter headers search. The basic strategy is to look at headers in the _future_ of our current best filter header * Fix bug in sql query were we were doing max chainWork too early on block headers, we needed to filter out headers in our set and _then_ we get the max chain work * Add more unit tests
2025-01-18 21:34:39 +01:00 · 2020-08-29 08:13:38 -05:00 · 2020-08-29 08:13:38 -05:00 · d3af9c2ccb
commit d3af9c2ccb
parent 43ba2477b5
6 changed files with 261 additions and 12 deletions
--- a/chain-test/src/test/scala/org/bitcoins/chain/blockchain/ChainHandlerTest.scala
+++ b/chain-test/src/test/scala/org/bitcoins/chain/blockchain/ChainHandlerTest.scala
@ -587,6 +587,20 @@ class ChainHandlerTest extends ChainDbUnitTest {
    }
  }

+  it must "get best filter header with zero blockchains in memory" in {
+    chainHandler: ChainHandler =>
+      val noChainsChainHandler = chainHandler.copy(blockchains = Vector.empty)
+
+      val filterHeaderF = noChainsChainHandler.getBestFilterHeader()
+
+      for {
+        filterHeaderOpt <- filterHeaderF
+      } yield {
+        assert(filterHeaderOpt.isDefined)
+        assert(filterHeaderOpt.get == ChainUnitTest.genesisFilterHeaderDb)
+      }
+  }
+
  final def processHeaders(
      processorF: Future[ChainApi],
      headers: Vector[BlockHeader],
--- a/chain-test/src/test/scala/org/bitcoins/chain/models/CompactFilterHeaderDAOTest.scala
+++ b/chain-test/src/test/scala/org/bitcoins/chain/models/CompactFilterHeaderDAOTest.scala
@ -1,6 +1,17 @@
 package org.bitcoins.chain.models

-import org.bitcoins.testkit.chain.ChainDbUnitTest
+import org.bitcoins.core.api.chain.db.{BlockHeaderDb, CompactFilterHeaderDb}
+import org.bitcoins.core.protocol.blockchain.BlockHeader
+import org.bitcoins.testkit.chain.{
+  BlockHeaderHelper,
+  ChainDbUnitTest,
+  ChainTestUtil,
+  ChainUnitTest
+}
+import org.bitcoins.testkit.core.gen.{
+  BlockchainElementsGenerator,
+  CryptoGenerators
+}
 import org.scalatest.FutureOutcome

 class CompactFilterHeaderDAOTest extends ChainDbUnitTest {
@ -18,4 +29,120 @@ class CompactFilterHeaderDAOTest extends ChainDbUnitTest {
        assert(opt == None)
      }
  }
+
+  it must "create and read a filter header from the database" in {
+    filterHeaderDAO =>
+      val blockHeaderDAO = BlockHeaderDAO()
+      val blockHeaderDb =
+        BlockHeaderHelper.buildNextHeader(ChainTestUtil.regTestGenesisHeaderDb)
+      val blockHeaderDbF = blockHeaderDAO.create(blockHeaderDb)
+      val filterHeaderDb1F = for {
+        blockHeaderDb <- blockHeaderDbF
+      } yield {
+        randomFilterHeader(blockHeaderDb)
+      }
+
+      val createdF = filterHeaderDb1F.flatMap(filterHeaderDAO.create)
+
+      for {
+        headerDb <- createdF
+        original <- filterHeaderDb1F
+        fromDbOpt <- filterHeaderDAO.read(headerDb.hashBE)
+      } yield {
+        assert(fromDbOpt.isDefined)
+        assert(original == fromDbOpt.get)
+      }
+  }
+
+  it must "get the best filter header that has a block header associated with it" in {
+    filterHeaderDAO =>
+      val blockHeaderDAO = BlockHeaderDAO()
+      val blockHeaderDb = {
+        BlockHeaderHelper.buildNextHeader(ChainTestUtil.regTestGenesisHeaderDb)
+      }
+      val blockHeaderDbF = blockHeaderDAO.create(blockHeaderDb)
+      val filterHeaderDb1F = for {
+        blockHeaderDb <- blockHeaderDbF
+      } yield {
+        randomFilterHeader(blockHeaderDb)
+      }
+
+      val createdF = filterHeaderDb1F.flatMap(filterHeaderDAO.create)
+      for {
+        blockHeader <- blockHeaderDbF
+        _ <- createdF
+        headers = Vector(blockHeader)
+        found <- filterHeaderDAO.getBestFilterHeaderForHeaders(headers)
+        empty <- filterHeaderDAO.getBestFilterHeaderForHeaders(Vector.empty)
+      } yield {
+        assert(found.nonEmpty)
+        assert(empty.isEmpty)
+      }
+  }
+
+  it must "fail to find a filter header if the block header is not in the db" in {
+    filterHeaderDAO =>
+      val blockHeaderDb = {
+        BlockHeaderHelper.buildNextHeader(ChainTestUtil.regTestGenesisHeaderDb)
+      }
+      val headers = Vector(blockHeaderDb)
+
+      for {
+        resultOpt <- filterHeaderDAO.getBestFilterHeaderForHeaders(headers)
+      } yield {
+        assert(resultOpt.isEmpty)
+      }
+  }
+
+  it must "find the filter header with the heaviest work" in {
+    filterHeaderDAO =>
+      val blockHeaderDAO = BlockHeaderDAO()
+      val blockHeaderDbLightWork = {
+        BlockHeaderHelper.buildNextHeader(ChainTestUtil.regTestGenesisHeaderDb)
+      }
+
+      val blockHeaderDbHeavyWork = {
+        blockHeaderDbLightWork.copy(
+          chainWork =
+            blockHeaderDbLightWork.chainWork + 1,
+          hashBE = CryptoGenerators.doubleSha256Digest.sample.get.flip)
+      }
+      val headers = Vector(blockHeaderDbLightWork, blockHeaderDbHeavyWork)
+      val blockHeaderDbF = blockHeaderDAO.createAll(headers)
+      val filterHeaderDbLightWork = {
+        randomFilterHeader(blockHeaderDbLightWork)
+      }
+
+      val filterHeaderDbHeavyWork = {
+        randomFilterHeader(blockHeaderDbHeavyWork)
+      }
+
+      val filterHeaders =
+        Vector(filterHeaderDbLightWork, filterHeaderDbHeavyWork)
+
+      val createdF = for {
+        _ <- blockHeaderDbF
+        created <- filterHeaderDAO.createAll(filterHeaders)
+      } yield created
+
+      for {
+        _ <- createdF
+        found <- filterHeaderDAO.getBestFilterHeader
+      } yield {
+        assert(found.nonEmpty)
+        assert(found.get == filterHeaderDbHeavyWork)
+      }
+  }
+
+  private def randomFilterHeader(
+      blockHeader: BlockHeaderDb): CompactFilterHeaderDb = {
+    CompactFilterHeaderDb(
+      CryptoGenerators.doubleSha256Digest.sample.get.flip,
+      filterHashBE = CryptoGenerators.doubleSha256Digest.sample.get.flip,
+      previousFilterHeaderBE =
+        CryptoGenerators.doubleSha256Digest.sample.get.flip,
+      blockHashBE = blockHeader.hashBE,
+      blockHeader.height
+    )
+  }
 }
--- a/chain/src/main/scala/org/bitcoins/chain/blockchain/BaseBlockChain.scala
+++ b/chain/src/main/scala/org/bitcoins/chain/blockchain/BaseBlockChain.scala
@ -88,6 +88,10 @@ private[blockchain] trait BaseBlockChain extends SeqWrapper[BlockHeaderDb] {

    loop(0)
  }
+
+  override def toString: String = {
+    s"BaseBlockchain(tip=${tip},last=${last},length=${length})"
+  }
 }

 private[blockchain] trait BaseBlockChainCompObject
--- a/chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala
+++ b/chain/src/main/scala/org/bitcoins/chain/blockchain/ChainHandler.scala
@ -374,7 +374,74 @@ case class ChainHandler(

  /** @inheritdoc */
  override def getBestFilterHeader(): Future[Option[CompactFilterHeaderDb]] = {
-    filterHeaderDAO.getBestFilterHeader
+    val bestFilterHeadersInChains: Vector[
+      Future[Option[CompactFilterHeaderDb]]] = {
+      blockchains.map { blockchain =>
+        filterHeaderDAO.getBestFilterHeaderForHeaders(blockchain.toVector)
+      }
+    }
+
+    val filterHeadersOptF: Future[Vector[Option[CompactFilterHeaderDb]]] = {
+      Future.sequence(bestFilterHeadersInChains)
+    }
+
+    for {
+      filterHeaders <- filterHeadersOptF
+      flattened = filterHeaders.flatten
+      result <-
+        if (flattened.isEmpty) {
+          bestFilterHeaderSearch()
+        } else {
+          Future.successful(Some(flattened.maxBy(_.height)))
+        }
+    } yield {
+      result
+    }
+  }
+
+  /**
+    * This method retrieves the best [[CompactFilterHeaderDb]] from the database
+    * without any blockchain context, and then uses the [[CompactFilterHeaderDb.blockHashBE]]
+    * to query our block headers database looking for a filter header that is in the best chain
+    * @return
+    */
+  private def bestFilterHeaderSearch(): Future[
+    Option[CompactFilterHeaderDb]] = {
+    val bestFilterHeaderOptF = filterHeaderDAO.getBestFilterHeader
+
+    //get best blockchain around our latest filter header
+    val blockchainF: Future[Blockchain] = {
+      for {
+        bestFilterHeaderOpt <- bestFilterHeaderOptF
+        blockchains <- {
+          bestFilterHeaderOpt match {
+            case Some(bestFilterHeader) =>
+              //get blockchains from our current best filter header to
+              //the next POW of interval, this should be enough to determine
+              //what is the best chain!
+              blockHeaderDAO.getBlockchainsBetweenHeights(
+                from =
+                  bestFilterHeader.height - chainConfig.chain.difficultyChangeInterval,
+                to =
+                  bestFilterHeader.height + chainConfig.chain.difficultyChangeInterval)
+            case None =>
+              Future.successful(Vector.empty)
+          }
+        }
+      } yield {
+        blockchains.maxBy(_.tip.chainWork)
+      }
+    }
+
+    val filterHeadersOptF: Future[Option[CompactFilterHeaderDb]] = {
+      for {
+        blockchain <- blockchainF
+        bestHeadersForChain <-
+          filterHeaderDAO.getBestFilterHeaderForHeaders(blockchain.toVector)
+      } yield bestHeadersForChain
+    }
+
+    filterHeadersOptF
  }

  /** @inheritdoc */
--- a/chain/src/main/scala/org/bitcoins/chain/models/CompactFilterHeaderDAO.scala
+++ b/chain/src/main/scala/org/bitcoins/chain/models/CompactFilterHeaderDAO.scala
@ -1,7 +1,7 @@
 package org.bitcoins.chain.models

 import org.bitcoins.chain.config.ChainAppConfig
-import org.bitcoins.core.api.chain.db.CompactFilterHeaderDb
+import org.bitcoins.core.api.chain.db.{BlockHeaderDb, CompactFilterHeaderDb}
 import org.bitcoins.crypto.DoubleSha256DigestBE
 import org.bitcoins.db.DatabaseDriver.{PostgreSQL, SQLite}
 import org.bitcoins.db.{CRUD, SlickUtil}
@ -123,14 +123,36 @@ case class CompactFilterHeaderDAO()(implicit
    query
  }

+  /** Fetches the best filter header from the database _without_ context
+    * that it's actually in our best blockchain. For instance, this filter header could be
+    * reorged out for whatever reason.
+    * @see https://github.com/bitcoin-s/bitcoin-s/issues/1919#issuecomment-682041737
+    */
  def getBestFilterHeader: Future[Option[CompactFilterHeaderDb]] = {
+    val blockHeaderDAO = BlockHeaderDAO()
+    val blockchainsF = blockHeaderDAO.getBlockchains()
+    blockchainsF.flatMap(blockchains =>
+      getBestFilterHeaderForHeaders(blockchains.flatten))
+  }
+
+  /** This looks for best filter headers whose [[CompactFilterHeaderDb.blockHashBE]] are associated with the given
+    * [[BlockHeaderDb.hashBE]] given as a parameter.
+    */
+  def getBestFilterHeaderForHeaders(
+      headers: Vector[BlockHeaderDb]): Future[Option[CompactFilterHeaderDb]] = {
+    val hashes = headers.map(_.hashBE)
    val join = table
      .join(blockHeaderTable)
      .on(_.blockHash === _.hash)

-    val maxQuery = join.map(_._2.chainWork).max
-
-    val query = join.filter(_._2.chainWork === maxQuery).take(1).map(_._1)
+    val query = join
+      .filter {
+        case (filterTable, _) =>
+          filterTable.blockHash.inSet(hashes)
+      }
+      .sortBy(_._2.chainWork.desc)
+      .take(1)
+      .map(_._1)

    for {
      filterOpt <-
@ -140,4 +162,20 @@ case class CompactFilterHeaderDAO()(implicit
    } yield filterOpt
  }

+  def getBetweenHeights(
+      from: Int,
+      to: Int): Future[Vector[CompactFilterHeaderDb]] = {
+    val query = getBetweenHeightsQuery(from, to)
+    safeDatabase.runVec(query)
+  }
+
+  def getBetweenHeightsQuery(
+      from: Int,
+      to: Int): profile.StreamingProfileAction[
+    Seq[CompactFilterHeaderDb],
+    CompactFilterHeaderDb,
+    Effect.Read] = {
+    table.filter(header => header.height >= from && header.height <= to).result
+  }
+
 }
--- a/node/src/main/scala/org/bitcoins/node/networking/peer/DataMessageHandler.scala
+++ b/node/src/main/scala/org/bitcoins/node/networking/peer/DataMessageHandler.scala
@ -302,20 +302,19 @@ case class DataMessageHandler(
      stopHash = stopHash)

  private def sendFirstGetCompactFilterHeadersCommand(
-      peerMsgSender: PeerMessageSender): Future[Boolean] =
+      peerMsgSender: PeerMessageSender): Future[Boolean] = {
    for {
-      filterHeaderCount <- chainApi.getFilterHeaderCount()
-      highestFilterHeaderOpt <-
+      bestFilterHeaderOpt <-
        chainApi
-          .getFilterHeadersAtHeight(filterHeaderCount)
-          .map(_.headOption)
+          .getBestFilterHeader()
      highestFilterBlockHash =
-        highestFilterHeaderOpt
+        bestFilterHeaderOpt
          .map(_.blockHashBE)
          .getOrElse(DoubleSha256DigestBE.empty)
      res <- sendNextGetCompactFilterHeadersCommand(peerMsgSender,
                                                    highestFilterBlockHash)
    } yield res
+  }

  private def sendNextGetCompactFilterCommand(
      peerMsgSender: PeerMessageSender,